Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Give a mixed NVIDIA plus AMD host a way to ask for the ROCm torch stack (#10474)
Daniel Hanb78f334a28e0
diff --git a/install.sh b/install.shindex 783ffbfef97..13ee08d39a1 100755--- a/install.sh+++ b/install.sh@@ -3800,15 +3800,31 @@ _ensure_rocm_probe_env() {fi}+# Whether this run was TOLD to install ROCm torch, whatever else the host has: the automatic+# profile stops probing AMD once CUDA is usable, leaving a mixed host no route (#10450). One+# torch install serves one vendor, so this SWAPS the stack.+# Mirrors install_python_stack._rocm_torch_explicitly_requested; keep the two in step.+_rocm_torch_explicitly_requested() {+ # Trimmed, because the Python twin reads the same variable through .strip(): untrimmed,+ # " true " exports an authoritative CUDA backend the Python half then cannot override.+ case "$(printf '%s' "${UNSLOTH_FORCE_ROCM_TORCH:-}" \+ | sed 's/^[[:space:]]*//; s/[[:space:]]*$//' | tr '[:upper:]' '[:lower:]')" in+ 1|true|yes|on) return 0 ;;+ *) return 1 ;;+ esac+}+# Whether ROCm can see an AMD GPU. A usable NVIDIA card wins by default; a diagnosis that-# has already established the run opens AMD nodes passes "ignore-nvidia" to skip that.+# has already established the run opens AMD nodes passes "ignore-nvidia" to skip that, and an+# explicit UNSLOTH_FORCE_ROCM_TORCH request relaxes it for every caller.# The veto is inside this function rather than in a wrapper, because every tests/sh harness# lifts probes one function at a time by name (sed -n '/^_name()/,/^}/p'), and a wrapper# whose helper is not also lifted calls nothing: the ROCm branch would then fall silently# through to the CPU wheel index._has_amd_rocm_gpu() {_ensure_rocm_probe_env- if [ "${1:-}" != "ignore-nvidia" ] && _has_usable_nvidia_gpu; then+ if [ "${1:-}" != "ignore-nvidia" ] && _has_usable_nvidia_gpu && \+ ! _rocm_torch_explicitly_requested; thenreturn 1fiif command -v rocminfo >/dev/null 2>&1 && \@@ -3826,6 +3842,269 @@ _has_amd_rocm_gpu() {return 1}+# AMD silicon this host can point at WITHOUT trusting a declared arch:+# _infer_linux_amd_gfx_arch returns UNSLOTH_ROCM_GFX_ARCH before it looks at hardware, so it+# cannot answer "is there a card" and a stale one would hand a working CUDA stack to no GPU.+_amd_hardware_corroborated() {+ _amd_gpu_present_via_pci && return 0+ # WSL enumerates no PCI display device, and neither /dev/dxg (an NVIDIA passthrough creates+ # it too) nor a leftover librocdxg names a vendor: beside an NVIDIA card the runtime must+ # name an agent itself, hence "physical" mode.+ if [ -e /dev/dxg ] || grep -qi microsoft /proc/version 2>/dev/null; then+ for _ahc_d in /opt/rocm/lib /opt/rocm/lib64 /opt/rocm-*/lib /opt/rocm-*/lib64; do+ { [ -e "$_ahc_d/librocdxg.so" ] || [ -e "$_ahc_d/librocdxg.so.1" ]; } || continue+ _has_usable_nvidia_gpu || return 0+ [ -n "$(_probe_amd_gfx_arch physical 2>/dev/null)" ] && return 0+ return 1+ done+ fi+ return 1+}++# Whether ANY index this installer can pick carries kernels for this arch. An arch in neither+# (gfx1010, RDNA 1) must never depose a card that can.+# Mirrors _gfx_has_a_wheel_route / _GENERIC_ROCM_WHEEL_GFX in install_python_stack.py.+_amd_gfx_has_wheel_route() {+ # A per-arch index carries its arch whatever the generic tag resolves to, so it answers first.+ _amd_arch_index_family_for_gfx "$1" >/dev/null 2>&1 && return 0+ case "$1" in+ gfx900|gfx906|gfx908|gfx90a|gfx942|gfx950) : ;;+ gfx1030|gfx1100|gfx1101|gfx1102|gfx1150|gfx1151|gfx1200|gfx1201) : ;;+ *) return 1 ;;+ esac+ _amd_generic_tag_carries_gfx "$1"+}++# Whether the generic wheel THIS host resolves carries kernels for an arch whose only route is+# that wheel. Tag comes from the installed ROCm version, so a stale /opt/rocm resolves a wheel+# predating the card; unreadable answers NO. Mirrors _GENERIC_WHEEL_GFX_MIN_ROCM (python twin).+_amd_generic_tag_carries_gfx() {+ case "$1" in+ gfx950|gfx1150|gfx1151) _agtc_min_major=7; _agtc_min_minor=0 ;;+ gfx1200|gfx1201) _agtc_min_major=6; _agtc_min_minor=4 ;;+ *) return 0 ;;+ esac+ _agtc_tag=$(_detect_rocm_version_tag 2>/dev/null) || _agtc_tag=""+ _agtc_ver=${_agtc_tag#rocm}+ _agtc_major=$(printf '%s' "$_agtc_ver" | cut -d. -f1)+ _agtc_minor=$(printf '%s' "$_agtc_ver" | cut -d. -f2)+ case "$_agtc_major$_agtc_minor" in+ ''|*[!0-9]*) return 1 ;;+ esac+ [ "$_agtc_major" -gt "$_agtc_min_major" ] && return 0+ [ "$_agtc_major" -eq "$_agtc_min_major" ] && [ "$_agtc_minor" -ge "$_agtc_min_minor" ] && return 0+ return 1+}++# True when a set mask exposes NO GPU at either layer. ROCr filters BENEATH HIP, and+# CUDA_VISIBLE_DEVICES is HIP's alias, read only when HIP is unset. ${VAR+x} not ${VAR:-}: a+# SET-but-empty mask hides every device. Mirrors _visible_masks_select_no_gpu.+_amd_visible_masks_select_no_gpu() {+ if [ -n "${HIP_VISIBLE_DEVICES+x}" ]; then+ _avm_hip=HIP_VISIBLE_DEVICES+ else+ _avm_hip=CUDA_VISIBLE_DEVICES+ fi+ for _avm_var in ROCR_VISIBLE_DEVICES "$_avm_hip"; do+ eval "_avm_set=\${$_avm_var+x}"+ eval "_avm_val=\${$_avm_var:-}"+ [ -n "$_avm_set" ] || continue+ case "$(printf '%s' "$_avm_val" | tr -d '[:space:]')" in+ ''|-1) return 0 ;;+ esac+ done+ return 1+}++# One mask layer applied to a per-DEVICE list, in mask order, never deduplicated. Survivors are+# the PREFIX of resolvable ordinals, since CUDA and HIP stop at the first entry naming no device.+# A third argument of "rocr" also ends the prefix at a REPEATED ordinal, which ROCr terminates on+# (ROCR-Runtime, core/inc/amd_filter_device.h); clr documents no such rule for HIP.+_amd_mask_survivors() {+ # The mask arrives through the environment, not `awk -v`: an assignment operand is+ # ESCAPE-PROCESSED, so HIP_VISIBLE_DEVICES='\061' became the ordinal 1 here while the Python+ # twin rejects it at int().+ printf '%s\n' "$1" | _ams_vis="$2" _ams_layer="${3:-}" awk '+ BEGIN { vis = ENVIRON["_ams_vis"]; layer = ENVIRON["_ams_layer"] }+ NF { vals[n++] = $0 }+ END {+ count = split(vis, want, ",")+ for (i = 1; i <= count; i++) {+ gsub(/^[ \t]+|[ \t]+$/, "", want[i])+ if (want[i] !~ /^[0-9]+$/) break+ idx = want[i] + 0+ if (idx >= n) break+ if (layer == "rocr" && (idx in seen)) break+ seen[idx] = 1+ print vals[idx]+ }+ }'+}++# The single arch the runtime will hand torch, empty when the masks resolve nothing. Resolved+# here, not by a probe: rocminfo honours only ROCR_VISIBLE_DEVICES and amd-smi neither. The+# layers COMPOSE, ROCr filtering and renumbering beneath HIP, and the FIRST survivor wins.+_amd_runtime_gfx_target() {+ _argt_list="$1"+ if [ -n "${ROCR_VISIBLE_DEVICES:-}" ]; then+ _argt_list=$(_amd_mask_survivors "$_argt_list" "$ROCR_VISIBLE_DEVICES" rocr)+ [ -n "$_argt_list" ] || return 0+ fi+ # CUDA_VISIBLE_DEVICES is the HIP alias and clr reads it only when HIP itself is unset.+ _argt_hip="${HIP_VISIBLE_DEVICES:-}"+ if [ -z "${HIP_VISIBLE_DEVICES+x}" ]; then+ _argt_hip="${CUDA_VISIBLE_DEVICES:-}"+ fi+ if [ -n "$_argt_hip" ]; then+ _argt_list=$(_amd_mask_survivors "$_argt_list" "$_argt_hip")+ [ -n "$_argt_list" ] || return 0+ fi+ printf '%s\n' "$_argt_list" | awk 'NF { print; exit }'+}++# The AMD integrated GPUs that shadow a discrete card by enumerating ahead of it. Mirror of+# _SHADOWING_INTEGRATED_GFX, held to it by tests/studio/install/test_rocm_arch_table_parity.py:+# one installer having this table and not the other is the shape of #7264 / #7277 / #7293.+_amd_gfx_is_shadowing_integrated() {+ case "$1" in+ gfx90c|gfx1013|gfx1033|gfx1035|gfx1036|gfx1103|gfx1153) return 0 ;;+ esac+ return 1+}++# The card to install for when enumeration put an integrated GPU first (#7776): one arch picks+# the wheel family, so letting the APU decide strands the discrete card. gfx906 is never a+# candidate, since naming it on a mixed host strands BOTH cards.+_amd_prefer_discrete_gfx() {+ _apdg_devs="$1"+ _apdg_sel="$2"+ if ! _amd_gfx_is_shadowing_integrated "$_apdg_sel"; then+ printf '%s' "$_apdg_sel"+ return 0+ fi+ if [ -n "${HIP_VISIBLE_DEVICES+x}" ] || [ -n "${ROCR_VISIBLE_DEVICES+x}" ] || \+ [ -n "${CUDA_VISIBLE_DEVICES+x}" ]; then+ printf '%s' "$_apdg_sel"+ return 0+ fi+ _apdg_others=$(printf '%s\n' "$_apdg_devs" | awk 'NF' | while IFS= read -r _apdg_g; do+ _amd_gfx_is_shadowing_integrated "$_apdg_g" && continue+ [ "$_apdg_g" = gfx906 ] && continue+ printf '%s\n' "$_apdg_g"+ done)+ _apdg_pick=$(printf '%s\n' "$_apdg_others" | awk 'NF' | while IFS= read -r _apdg_g; do+ _amd_gfx_has_wheel_route "$_apdg_g" && printf '%s\n' "$_apdg_g"+ done | awk 'NF { print; exit }')+ if [ -z "$_apdg_pick" ] && ! _amd_gfx_has_wheel_route "$_apdg_sel"; then+ _apdg_pick=$(printf '%s\n' "$_apdg_others" | awk 'NF { print; exit }')+ fi+ [ -n "$_apdg_pick" ] || _apdg_pick="$_apdg_sel"+ printf '%s' "$_apdg_pick"+}++# Whether the request has something to swap TO: a corroborated card whose arch an index can+# serve. Presence is not that bar -- gfx1010 is present with no route, and gfx906 drops when a+# second AMD arch is present.+_amd_request_has_a_wheel_route() {+ # The card this run hands torch, published for get_torch_index_url's miscomputing-arch gate,+ # which has only the unmasked inventory to judge on. Cleared on ENTRY so a previous call can+ # never answer for this one.+ _AMD_REQUEST_TARGET_GFX=""+ # Only a PROBE-resolved target may clear a safety gate, or a declared gfx1030 would exempt a+ # physical gfx1033 from the Van Gogh gate. Declared still routes (the Strix workaround+ # declares gfx1100 on a gfx1151), it just cannot vouch for the machine.+ _AMD_REQUEST_TARGET_SOURCE=""+ # A mask exposing no device is a deliberate no-GPU selection, not a detection miss.+ _amd_visible_masks_select_no_gpu && return 1+ # A DECLARED arch decides when there is one, since get_torch_index_url installs from it.+ # Corroborated first, because the variable answers on a host with no AMD GPU at all.+ _arwr_decl=$(printf '%s' "${UNSLOTH_ROCM_GFX_ARCH:-}" \+ | tr '[:upper:]' '[:lower:]' | sed 's/:.*$//' | tr -d '[:space:]')+ if [ -n "$_arwr_decl" ]; then+ _amd_hardware_corroborated || _kfd_gfx_targets 2>/dev/null | grep -q . || return 1+ # The arch names what to BUILD for, not whether the runtime exposes a device to build it+ # for. Asked only when a mask is set AND the device list is knowable: an unknowable one+ # leaves the declared arch alone rather than declining on no evidence.+ if [ -n "${HIP_VISIBLE_DEVICES+x}" ] || [ -n "${ROCR_VISIBLE_DEVICES+x}" ] || \+ [ -n "${CUDA_VISIBLE_DEVICES+x}" ]; then+ _arwr_dd=$(_amd_ordered_gfx_devices 2>/dev/null | sed 's/:.*$//' \+ | tr '[:upper:]' '[:lower:]' | awk 'NF')+ [ -n "$_arwr_dd" ] || _arwr_dd=$(_kfd_gfx_targets 2>/dev/null | sed 's/:.*$//' \+ | tr '[:upper:]' '[:lower:]' | awk 'NF')+ if [ -n "$_arwr_dd" ] && [ -z "$(_amd_runtime_gfx_target "$_arwr_dd")" ]; then+ return 1+ fi+ fi+ if _amd_gfx_has_wheel_route "$_arwr_decl"; then+ _AMD_REQUEST_TARGET_GFX="$_arwr_decl"+ _AMD_REQUEST_TARGET_SOURCE=declared+ return 0+ fi+ return 1+ fi+ _arwr_all=$(_probe_amd_gfx_arch physical 2>/dev/null || true)+ [ -n "$_arwr_all" ] || _arwr_all=$(_kfd_gfx_targets 2>/dev/null || true)+ if [ -z "$_arwr_all" ]; then+ # Runtime-less but inferable, which a pure-AMD host is already served on. Requiring a working+ # ROCm runtime would answer differently for the same silicon by whether an NVIDIA card sits+ # beside it.+ _amd_hardware_corroborated || return 1+ _arwr_all=$(_infer_linux_amd_gfx_arch 2>/dev/null || true)+ fi+ _arwr_archs=$(printf '%s\n' "$_arwr_all" | sed 's/:.*$//' \+ | tr '[:upper:]' '[:lower:]' | awk 'NF')+ [ -n "$_arwr_archs" ] || return 1+ # The PHYSICAL count, before any mask narrows the list: gfx906's rocm6.3 tag opens only when+ # it is the sole arch, and the reroute granting it inspects the unmasked inventory.+ _arwr_count=$(printf '%s\n' "$_arwr_archs" | sort -u | wc -l | tr -d ' ')+ # The one card this run will hand torch; empty when the masks named something unresolvable.+ # Fails CLOSED, since a wrong yes replaces a working CUDA stack with kernel-less wheels.+ _arwr_devs=$(_amd_ordered_gfx_devices 2>/dev/null | sed 's/:.*$//' \+ | tr '[:upper:]' '[:lower:]' | awk 'NF')+ if [ -z "$_arwr_devs" ]; then+ # rocminfo is the only per-device source the masks index. amd-smi enumerates in KFD DISCOVERY+ # order while the kernel topology IS HIP's and ROCr's: take it when it describes this machine.+ _arwr_kfd=$(_kfd_gfx_targets 2>/dev/null | sed 's/:.*$//' \+ | tr '[:upper:]' '[:lower:]' | awk 'NF')+ if [ -n "$_arwr_kfd" ] && \+ [ "$(printf '%s\n' "$_arwr_kfd" | awk 'NF' | wc -l | tr -d ' ')" \+ = "$(printf '%s\n' "$_arwr_archs" | awk 'NF' | wc -l | tr -d ' ')" ]; then+ _arwr_devs="$_arwr_kfd"+ fi+ fi+ if [ -n "$_arwr_devs" ]; then+ _arwr_sel=$(_amd_runtime_gfx_target "$_arwr_devs")+ elif [ "$_arwr_count" -eq 1 ]; then+ # One arch on the whole host: every ordinal names it, so ordering cannot change the answer.+ _arwr_sel=$(_amd_runtime_gfx_target "$_arwr_archs")+ else+ # Unlike adapters with no order the ordinals fit: which card the runtime hands torch is+ # unanswerable here, so fail closed.+ return 1+ fi+ [ -n "$_arwr_sel" ] || return 1+ # Enumeration order is not a choice of card. The flat inventory only stands in where the+ # count is 1 and no preference can apply anyway.+ _arwr_pref="$_arwr_devs"+ [ -n "$_arwr_pref" ] || _arwr_pref="$_arwr_archs"+ _arwr_sel=$(_amd_prefer_discrete_gfx "$_arwr_pref" "$_arwr_sel")+ [ "$_arwr_sel" = gfx906 ] && [ "$_arwr_count" -gt 1 ] && return 1+ _amd_gfx_has_wheel_route "$_arwr_sel" || return 1+ _AMD_REQUEST_TARGET_GFX="$_arwr_sel"+ _AMD_REQUEST_TARGET_SOURCE=probe+ return 0+}++# One place answers "does the NVIDIA card still win here", so index selection and the per-arch+# reroutes cannot disagree: the reroutes probe afresh at top level, so get_torch_index_url+# clearing its own _nvidia_detected never reached them (#10450).+_nvidia_gpu_wins_over_amd() {+ _has_usable_nvidia_gpu || return 1+ if _rocm_torch_explicitly_requested && _amd_request_has_a_wheel_route; then+ return 1+ fi+ return 0+}# Returns 0 if an AMD display GPU is on the PCI bus even when ROCm cannot use it (a Strix Halo iGPU with no /dev/kfd). Only sharpens the "no GPU detected" hint. vendor 0x1002 = AMD/ATI; class 0x03* = display controller._amd_gpu_present_via_pci() {[ -d /sys/bus/pci/devices ] || return 1@@ -4524,6 +4803,15 @@ _probe_amd_gfx_arch() {printf '%s\n' "$_pg"}+# One gfx per GPU in ROCr enumeration order, the order a mask indexes. _probe_amd_gfx_arch+# cannot serve: rocminfo names a target in BOTH "Name:" and "ISA Info", so a flat grep returns+# two rows per card. Twin of install_python_stack._detect_amd_gfx_codes(dedup = False).+_amd_ordered_gfx_devices() {+ _ensure_rocm_probe_env+ command -v rocminfo >/dev/null 2>&1 || return 0+ (unset ROCR_VISIBLE_DEVICES HIP_VISIBLE_DEVICES HSA_OVERRIDE_GFX_VERSION; rocminfo 2>/dev/null) \+ | _rocminfo_gpu_records | sed 's/|.*$//' | awk 'NF'+}# Classify the physical NVIDIA inventory for a cu126 fallback: "cu126" when it covers every GPU, "uncovered" for an incompatible mix, empty when no fallback is needed or the inventory is unreadable. CUDA_VISIBLE_DEVICES is ignored because the wheel must support the host. Shared decision with install.ps1 / setup.ps1 / install_python_stack.py._nvidia_cu126_verdict() {# $2: capabilities already read from the driver library, one per line, when nvidia-smi cannot.@@ -4700,6 +4988,11 @@ _detect_rocm_version_tag() {# On CPU-only machines this returns the cpu index, avoiding the solver# dead-end where --torch-backend=auto resolves to unsloth==2024.8.get_torch_index_url() {+ # Cleared on ENTRY, not only in the helper that sets them: the helper runs only when NVIDIA is+ # detected AND the request is set, and the CUDA restore calls this a second time from a shell+ # the reroutes have populated. A stale gfx1100/probe pair clears the Van Gogh gate on gfx1033.+ _AMD_REQUEST_TARGET_GFX=""+ _AMD_REQUEST_TARGET_SOURCE=""_base="${UNSLOTH_PYTORCH_MIRROR:-https://download.pytorch.org/whl}"_base="${_base%/}"# An explicit override skips ALL GPU probing: the URL is verbatim, _FAMILY is its leaf.@@ -4730,6 +5023,15 @@ get_torch_index_url() {_smi="/usr/bin/nvidia-smi"fifi+ # Here rather than only in _has_amd_rocm_gpu, because this index is exported as+ # UNSLOTH_TORCH_BACKEND, on which _ensure_rocm_torch returns immediately for "cuda":+ # relaxing the AMD probe alone swapped nothing (#10450).+ if [ "$_nvidia_detected" -eq 1 ] && _rocm_torch_explicitly_requested && \+ _amd_request_has_a_wheel_route; then+ echo "[INFO] UNSLOTH_FORCE_ROCM_TORCH is set and an AMD GPU is present -- selecting ROCm PyTorch over CUDA." >&2+ echo "[INFO] One torch install serves one vendor: the NVIDIA card will not be available to training until this is unset and the installer re-run." >&2+ _nvidia_detected=0+ fiif [ "$_nvidia_detected" -eq 0 ]; thencase "$(uname -m)" inx86_64|amd64) : ;;@@ -4780,13 +5082,29 @@ get_torch_index_url() {[ -n "$_amd_gfx_gate_probe" ] || _amd_gfx_gate_probe="$_amd_gfx_probe"_amd_gfx_tokens=" $(printf '%s\n' "$_amd_gfx_gate_probe" | sed 's/:.*$//' \| tr '[:upper:]' '[:lower:]' | tr '\n' ' ')"+ _amd_gfx_bad_arch=falsecase "$_amd_gfx_tokens" in- *" gfx1033 "*)- echo "[WARN] AMD gfx1033 (Van Gogh) computes incorrect results under ROCm -- installing CPU-only PyTorch." >&2- echo "[WARN] ROCm wheels install on it but training diverges to NaN and gradcheck fails; forward math is fine." >&2- echo "[WARN] Details: studio/ROCM_RDNA2_APU.md. Override with UNSLOTH_TORCH_INDEX_URL if you want ROCm anyway." >&2- echo "$_base/cpu"; return ;;+ *" gfx1033 "*) _amd_gfx_bad_arch=true ;;esac+ # PRESENCE is the rule while the selected card is unknown. An honoured request has resolved+ # the one card this run hands torch, so answering on a sibling it hid took the cpu index for+ # a routable gfx1100. Probe-resolved only.+ case "${_AMD_REQUEST_TARGET_GFX:-}" in+ ""|gfx1033) : ;;+ *)+ # if/fi rather than `[ ... ] &&`: under set -e a failing test as the last+ # command of the arm would abort the installer.+ if [ "${_AMD_REQUEST_TARGET_SOURCE:-}" = probe ]; then+ _amd_gfx_bad_arch=false+ fi+ ;;+ esac+ if [ "$_amd_gfx_bad_arch" = true ]; then+ echo "[WARN] AMD gfx1033 (Van Gogh) computes incorrect results under ROCm -- installing CPU-only PyTorch." >&2+ echo "[WARN] ROCm wheels install on it but training diverges to NaN and gradcheck fails; forward math is fine." >&2+ echo "[WARN] Details: studio/ROCM_RDNA2_APU.md. Override with UNSLOTH_TORCH_INDEX_URL if you want ROCm anyway." >&2+ echo "$_base/cpu"; return+ fi# end of the miscomputing-arch gate -- tests/sh/test_rocm_bad_arch_gate.sh lifts# the block between the header comment above and this line, so keep both exact._rocm_tag=""@@ -4899,6 +5217,16 @@ _torch_index_url_leaf() {printf '%s' "${_tl_u##*/}" | tr '[:upper:]' '[:lower:]'}+# Did the resolution land on a ROCm index? The FINAL leaf only, so a mirror whose BASE path+# contains rocm or gfx cannot read as one. Broader than _is_pip_rocm_family_leaf on purpose,+# which declines the Radeon repo leaf rocm-rel-6.4.+_torch_index_url_is_rocm() {+ case "$(_torch_index_url_leaf "${1:-}")" in+ rocm*|gfx*) return 0 ;;+ *) return 1 ;;+ esac+}+# True for an EXACT ROCm family leaf; one that merely starts with rocm/gfx is a custom pin._is_pip_rocm_family_leaf() {case "$1" in@@ -5399,7 +5727,7 @@ case "$TORCH_INDEX_URL" in*/cpu)if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \[ -z "${UNSLOTH_ROCM_GFX_ARCH:-}" ] && \- ! _has_usable_nvidia_gpu && _has_amd_rocm_gpu; then+ ! _nvidia_gpu_wins_over_amd && _has_amd_rocm_gpu; then_amd_probe_out=$(_probe_amd_gfx_arch)# HSA_OVERRIDE_GFX_VERSION=11.0.0 is the standard Strix Halo workaround, and ROCr then reports the spoofed gfx1100. The llama.cpp path corrects that further down, which is too late for this branch: both the wheel family and the exported arch are derived from this probe, so an uncorrected gfx1151 would take gfx110X-all wheels and export gfx1100 to setup.sh._amd_spoof_inferred=$(_infer_linux_amd_gfx_arch 2>/dev/null || true)@@ -5416,14 +5744,44 @@ case "$TORCH_INDEX_URL" in|| _amd_probed_family=""_amd_probed_gfx_first=$(_amd_sole_index_arch "$_amd_probe_out") \|| _amd_probed_gfx_first=""+ # Re-derived rather than read: _AMD_REQUEST_TARGET_GFX is assigned inside a command+ # substitution above, and the predicate is idempotent.+ _amd_reroute_target=""+ if _rocm_torch_explicitly_requested && _amd_request_has_a_wheel_route; then+ _amd_reroute_target="${_AMD_REQUEST_TARGET_GFX:-}"+ fi+ _amd_reroute_bad_arch=false# A measured-bad arch anywhere in the inventory disqualifies the whole family, at the source not at each consumer: gfx1033 shares gfx103X-all with gfx1030-gfx1036, so the shared-family arm below would otherwise rewrite the cpu index the gate just chose back to ROCm wheels.case " $(printf '%s\n' "$_amd_probe_out" | sed 's/:.*$//' \| tr '[:upper:]' '[:lower:]' | tr '\n' ' ')" in- *" gfx1033 "*)- echo "[WARN] AMD gfx1033 (Van Gogh) is in the probed inventory -- not routing torch to the shared $_amd_probed_family index (studio/ROCM_RDNA2_APU.md)." >&2- _amd_probed_family=""- _amd_probed_gfx_first="" ;;+ *" gfx1033 "*) _amd_reroute_bad_arch=true ;;+ esac+ # Probe-resolved only, exactly as the gate inside get_torch_index_url.+ case "$_amd_reroute_target" in+ ""|gfx1033) : ;;+ *)+ if [ "${_AMD_REQUEST_TARGET_SOURCE:-}" = probe ]; then+ _amd_reroute_bad_arch=false+ fi+ ;;esac+ if [ "$_amd_reroute_bad_arch" = true ]; then+ echo "[WARN] AMD gfx1033 (Van Gogh) is in the probed inventory -- not routing torch to the shared $_amd_probed_family index (studio/ROCM_RDNA2_APU.md)." >&2+ _amd_probed_family=""+ _amd_probed_gfx_first=""+ fi+ # Derived from the SELECTED card: _amd_agreed_index_family needs every physical GPU to+ # share a family and answers empty for a cross-family pair, which left the cpu index+ # unrewritten on exactly the host that resolved a routable target.+ if [ "$_amd_reroute_bad_arch" = false ] && [ -n "$_amd_reroute_target" ] && \+ [ "${_AMD_REQUEST_TARGET_SOURCE:-}" = probe ]; then+ _amd_target_family=$(_amd_arch_index_family_for_gfx "$_amd_reroute_target") \+ || _amd_target_family=""+ if [ -n "$_amd_target_family" ]; then+ _amd_probed_family="$_amd_target_family"+ _amd_probed_gfx_first="$_amd_reroute_target"+ fi+ fiif [ -n "${_amd_probed_family:-}" ] && \[ -z "$(_detect_rocm_version_tag 2>/dev/null)" ]; then_amd_no_rocm_version_reroute=true@@ -5432,7 +5790,7 @@ case "$TORCH_INDEX_URL" in;;esacif [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \- ! _has_usable_nvidia_gpu && \+ ! _nvidia_gpu_wins_over_amd && \{ [ -n "${UNSLOTH_ROCM_GFX_ARCH:-}" ] || ! _has_amd_rocm_gpu || \[ -z "$(_probe_amd_gfx_arch)" ] || \[ "${_amd_no_rocm_version_reroute:-false}" = true ]; } && \@@ -5524,6 +5882,21 @@ if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \esacfi+# The request buys a swap, never a downgrade. Asked of the RESOLVED index: the AMD branch+# answers cpu for a ROCm too old, an arch no index covers, or a non-x86_64 host, and leaving+# that would replace a working CUDA install with CPU torch.+if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \+ _rocm_torch_explicitly_requested && _has_usable_nvidia_gpu; then+ if ! _torch_index_url_is_rocm "$TORCH_INDEX_URL"; then+ _cuda_fallback_index=$(UNSLOTH_FORCE_ROCM_TORCH=0 get_torch_index_url)+ if [ -n "$_cuda_fallback_index" ] && \+ [ "$_cuda_fallback_index" != "$TORCH_INDEX_URL" ]; then+ echo "[WARN] UNSLOTH_FORCE_ROCM_TORCH is set, but no ROCm wheel index could be selected for this host." >&2+ echo "[WARN] Keeping the CUDA build rather than installing CPU PyTorch on a machine with a working NVIDIA GPU." >&2+ TORCH_INDEX_URL="$_cuda_fallback_index"+ fi+ fi+fi# Export the resolved torch backend ("cuda", "rocm" or "cpu") so setup.sh and install_python_stack.py know what was chosen here and can skip ROCm-specific repair steps. Classify on the FINAL path segment only: a custom UNSLOTH_PYTORCH_MIRROR whose base path happens to contain "rocm" or "gfx" must not mislabel a cu*/cpu index as ROCm (radeon repo URLs end in rocm-rel-X.Y/, Strix overrides in gfxNNNN/, so the trailing slash is stripped first). Lowercase the leaf so every gfx*/rocm*/cu* arm matches regardless of case (the canonical AMD RDNA4 leaf is gfx120X-all). CUDA is branded only on a real cu[0-9]* leaf, so a mirror leaf (/current) does NOT commit a CUDA backend; an unknown leaf leaves the var unset so the stack probes the GPU. Query and fragment are dropped first, then ALL trailing slashes, in lockstep with the shared _torch_index_url_leaf extractor._torch_index_leaf="${TORCH_INDEX_URL%%\?*}"_torch_index_leaf="${_torch_index_leaf%%#*}"@@ -5865,8 +6238,15 @@ fi_TAURI_GPU_BRANCH=$(_tauri_gpu_branch "$_TAURI_TORCH_INDEX_FAMILY" "$_amd_gpu_radeon")tauri_diag_marker "$_TAURI_GPU_BRANCH" "$_TAURI_TORCH_INDEX_FAMILY"+# ── GPU detection summary (mirrors install.ps1 step "gpu" block) ──-if _has_usable_nvidia_gpu; then+# Asked of the RESOLVED index, not the predicate that chose it: after the CUDA restore the+# request is still set and an AMD card still present, so the predicate says "AMD wins" while+# CUDA wheels are what gets installed. A PIN is exempt, because it names a wheel family rather+# than a card: on an NVIDIA host pinned to a ROCm index there is no AMD card to describe, and+# hiding the NVIDIA identity behind it reported hardware the machine does not have.+if _has_usable_nvidia_gpu && \+ { [ "$_torch_index_pinned" = true ] || ! _torch_index_url_is_rocm "$TORCH_INDEX_URL"; }; then_nv_banner_fieldsif [ -n "$_nv_name" ] && [ -n "$_nv_sm" ]; thenstep "gpu" "$_nv_name ($_nv_sm)"@@ -5878,7 +6258,7 @@ if _has_usable_nvidia_gpu; then# An `if`, not `[ ... ] && substep ...`: the AND-list form leaves a non-zero status# behind on the common path where there is no driver string to print.if [ -n "$_nv_driver" ]; then substep "Driver: $_nv_driver"; fi-elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then+elif _torch_index_url_is_rocm "$TORCH_INDEX_URL"; then# Probe gfx arch for the display label, honouring HIP_VISIBLE_DEVICES_ensure_rocm_probe_env_gpu_disp_gfx_all=""diff --git a/studio/backend/tests/test_amd_device_node_permissions.py b/studio/backend/tests/test_amd_device_node_permissions.pyindex 74b885b5267..5d2f65e9156 100644--- a/studio/backend/tests/test_amd_device_node_permissions.py+++ b/studio/backend/tests/test_amd_device_node_permissions.py@@ -4426,9 +4426,19 @@ def test_the_two_quoting_rules_are_the_same_rule(tmp_path):assert out.stdout == shlex.quote(value), (value, out.stdout)-# The two helpers every tests/sh ROCm harness lifts alongside _has_amd_rocm_gpu.+# The helpers every tests/sh ROCm harness lifts alongside _has_amd_rocm_gpu. Adding a name+# here is only honest once all five harnesses lift it too, which is what the test below the+# invariant checks: each name must appear in every harness that lifts the probe._ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT = frozenset(- {"_ensure_rocm_probe_env", "_has_usable_nvidia_gpu"}+ {+ "_ensure_rocm_probe_env",+ "_has_usable_nvidia_gpu",+ # Reads UNSLOTH_FORCE_ROCM_TORCH, the opt-in that lets a mixed NVIDIA+AMD host ask+ # for the ROCm stack (#10450). Unlifted it is an undefined command, which exits+ # non-zero, which reads as "not requested" -- the default answer, so a harness would+ # keep passing while the branch under test was never the one the flag selects.+ "_rocm_torch_explicitly_requested",+ })@@ -4457,6 +4467,27 @@ def test_the_rocm_probe_calls_nothing_the_shell_harnesses_do_not_lift():assert called <= _ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT, called+def test_the_harnesses_really_lift_every_name_the_allowlist_claims():+ """The control on the allowlist above. That test only fails when the probe grows a call,+ so widening the frozenset silences it whether or not the harnesses were updated -- which+ is the same silent failure by a shorter route. Check the harnesses themselves: every+ tests/sh file that lifts _has_amd_rocm_gpu must lift each allowlisted callee too."""+ sh_dir = Path(__file__).resolve().parents[3] / "tests" / "sh"+ harnesses = [+ path+ for path in sorted(sh_dir.glob("*.sh"))+ if "_has_amd_rocm_gpu" in path.read_text(encoding = "utf-8")+ ]+ # Guard the guard: a glob that matched nothing would pass this vacuously.+ assert len(harnesses) >= 5, [p.name for p in harnesses]+ for path in harnesses:+ source = path.read_text(encoding = "utf-8")+ # A harness that only stubs the probe (bad_arch_gate does this for some cases) still+ # lifts it elsewhere; asking for the name anywhere in the file is the right grain.+ for callee in sorted(_ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT):+ assert callee in source, f"{path.name} lifts _has_amd_rocm_gpu but not {callee}"++def test_that_check_sees_a_helper_the_harnesses_would_not_have():"""The control. Without it the test above could be passing because the scan matchesnothing at all, which is what a name-based scan usually does when it is wrong."""diff --git a/studio/backend/tests/test_rocm_torch_explicit_request.py b/studio/backend/tests/test_rocm_torch_explicit_request.pynew file mode 100644index 00000000000..db413444d12--- /dev/null+++ b/studio/backend/tests/test_rocm_torch_explicit_request.py@@ -0,0 +1,2684 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A mixed NVIDIA+AMD host must have SOME route to its AMD card.++Every installer stops probing AMD the moment nvidia-smi lists a GPU. That is the+right default -- CUDA is the better-supported stack and a false AMD positive would+swap a working install -- but it left a mixed host with no route at all: chat can be+pointed at Vulkan in Settings, and training reads whatever torch was installed, so+the AMD card is invisible to it. The only bypass was an index pin, which is+undocumented and names a wheel family rather than a preference (#10450).++``UNSLOTH_FORCE_ROCM_TORCH=1`` is the request, mirroring ``UNSLOTH_FORCE_VULKAN``+for the llama.cpp bundle and the ``backend == "rocm"`` re-probe in+``install_llama_prebuilt._route_to_vulkan_prebuilt``.++One torch install serves one vendor, so the request SWAPS the stack: these tests+assert it is honoured, and that it changes nothing at all unless it is set.+"""++from __future__ import annotations++import importlib.util+import os+import subprocess+import sys+import types+from pathlib import Path++import pytest++_STACK = Path(__file__).resolve().parents[2] / "install_python_stack.py"++[email protected](scope = "module")+def stack():+ """install_python_stack imported by path: it is a top-level installer script, not a package+ module, so the backend's own import path does not reach it."""+ spec = importlib.util.spec_from_file_location("_unsloth_install_python_stack", _STACK)+ module = importlib.util.module_from_spec(spec)+ sys.modules[spec.name] = module+ spec.loader.exec_module(module)+ return module+++class _StoppedAtProbe(BaseException):+ """Raised by the stubbed ROCm version probe so nothing past it runs."""++[email protected](autouse = True)+def _no_real_install(stack, monkeypatch):+ """No test in this file may reach an installation."""++ def _refuse(*args, **kwargs):+ pytest.fail(f"the test reached a real install: pip_install{args[:2]}")++ monkeypatch.setattr(stack, "pip_install", _refuse)+ monkeypatch.setattr(stack, "pip_install_try", _refuse)+++def _reaches_the_version_probe(stack, monkeypatch) -> bool:+ """Whether _ensure_rocm_torch gets as far as the ROCm version probe."""+ reached = {"ran": False}++ def _probe(*args, **kwargs):+ reached["ran"] = True+ raise _StoppedAtProbe++ monkeypatch.setattr(stack, "_detect_rocm_version", _probe)+ try:+ stack._ensure_rocm_torch()+ except _StoppedAtProbe:+ pass+ return reached["ran"]++[email protected]("value", ["1", "true", "TRUE", "yes", "on"])+def test_the_request_is_recognised(stack, monkeypatch, value):+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", value)+ assert stack._rocm_torch_explicitly_requested() is True++[email protected]("value", ["0", "false", "no", "", " "])+def test_anything_else_is_not_a_request(stack, monkeypatch, value):+ """Including the empty string: an exported-but-empty variable is how a shell passes "unset",+ and reading it as a request would swap a CUDA host's stack."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", value)+ assert stack._rocm_torch_explicitly_requested() is False+++def test_unset_is_not_a_request(stack, monkeypatch):+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)+ assert stack._rocm_torch_explicitly_requested() is False+++def test_an_nvidia_host_still_hides_the_amd_card_by_default(stack, monkeypatch):+ """The control. Every assertion below is only meaningful if the default holds."""+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ assert stack._has_rocm_gpu() is False+++def _rocminfo(monkeypatch, stack, output: str):+ """A host where rocminfo is on PATH and prints ``output``; amd-smi absent."""+ monkeypatch.setattr(+ stack.shutil, "which", lambda c: "/usr/bin/rocminfo" if c == "rocminfo" else None+ )+ monkeypatch.setattr(+ stack.subprocess,+ "run",+ lambda *a, **k: types.SimpleNamespace(returncode = 0, stdout = output),+ )+ # The sysfs fallback must not answer for the shelled probes above.+ monkeypatch.setattr(stack.os.path, "isdir", lambda p: False)+++def test_the_request_lets_the_amd_probe_run_on_a_mixed_host(stack, monkeypatch):+ """Fails before the fix: _has_rocm_gpu returned False on any NVIDIA host before consulting a+ single AMD probe, so nothing downstream could see the AMD card."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ _rocminfo(monkeypatch, stack, " Name: gfx1201")+ assert stack._has_rocm_gpu() is True+++def test_the_same_host_without_the_request_still_answers_no(stack, monkeypatch):+ """The pair to the test above, on an identical host: the ONLY difference is the variable, so a+ probe that started answering for some other reason fails here."""+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ _rocminfo(monkeypatch, stack, " Name: gfx1201")+ assert stack._has_rocm_gpu() is False+++def test_the_request_does_not_invent_a_card(stack, monkeypatch):+ """The request relaxes which vendor wins, not whether there is a card to serve."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ _rocminfo(monkeypatch, stack, " Name: gfx000") # CPU agent only+ assert stack._has_rocm_gpu() is False+++def _shell_function(name: str) -> str:+ """The text of one function from install.sh, by brace matching."""+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"+ lines = install_sh.read_text(encoding = "utf-8").splitlines()+ start = next(i for i, line in enumerate(lines) if line.startswith(f"{name}() {{"))+ depth = 0+ for end in range(start, len(lines)):+ depth += lines[end].count("{") - lines[end].count("}")+ if depth == 0:+ return "\n".join(lines[start : end + 1])+ raise AssertionError(f"unterminated function {name}")+++def _clean_env(extra: "dict | None" = None) -> dict:+ """The runner's environment minus the two things that decide these cases by themselves."""+ env = {+ k: v+ for k, v in os.environ.items()+ if not k.endswith("VISIBLE_DEVICES") and k != "UNSLOTH_ROCM_GFX_ARCH"+ }+ env.update(extra or {})+ return env+++def _bash(script: str, *, env: "dict | None" = None) -> str:+ """Run a lifted script under bash and return its stdout, failing with its own stderr."""+ out = subprocess.run(+ ["bash", "-c", script], capture_output = True, text = True, env = env or _clean_env()+ )+ assert out.returncode == 0, out.stderr+ return out.stdout.strip()+++# `command -v rocminfo` must not reach a real one on a ROCm build host, or these answers+# would depend on the machine running the suite. Each harness names its own inventory.+_rocminfo_stub = "rocminfo() { return 1; }"+++def _wheel_route_defs(*, arch_family: bool = True, rocminfo: str = _rocminfo_stub) -> "list[str]":+ """Everything _amd_request_has_a_wheel_route consults, lifted from install.sh."""+ return [+ _shell_function("_amd_hardware_corroborated"),+ *([_shell_function("_amd_arch_index_family_for_gfx")] if arch_family else []),+ _shell_function("_amd_gfx_has_wheel_route"),+ _shell_function("_amd_visible_masks_select_no_gpu"),+ _shell_function("_amd_generic_tag_carries_gfx"),+ _shell_function("_amd_mask_survivors"),+ _shell_function("_amd_runtime_gfx_target"),+ _shell_function("_amd_gfx_is_shadowing_integrated"),+ _shell_function("_amd_prefer_discrete_gfx"),+ _shell_function("_rocminfo_gpu_records"),+ _shell_function("_amd_ordered_gfx_devices"),+ "_ensure_rocm_probe_env() { :; }",+ rocminfo,+ _shell_function("_amd_request_has_a_wheel_route"),+ ]+++def _fake_rocminfo(devices: "list[str]") -> str:+ """A rocminfo whose output has the shape the real one has."""+ blocks = [+ "Agent 1",+ "*******",+ " Name: AMD Ryzen 9",+ " Marketing Name: AMD Ryzen 9",+ " Device Type: CPU",+ ]+ for _i, _gfx in enumerate(devices):+ blocks += [+ "*******",+ f"Agent {_i + 2}",+ "*******",+ f" Name: {_gfx}",+ " Marketing Name: AMD Radeon Graphics",+ " Device Type: GPU",+ " ISA Info:",+ " ISA 1",+ f" Name: amdgcn-amd-amdhsa--{_gfx}",+ ]+ body = "\n".join(blocks).replace("'", "")+ return "rocminfo() { cat <<'ROCMINFO_EOF'\n" + body + "\nROCMINFO_EOF\n}"+++def _index_url(env: str, stubs: str) -> str:+ """get_torch_index_url() under a stubbed host, returning the wheel index it picks."""+ script = "\n".join(+ [+ # Every case here describes a Linux x86_64 host, so say so rather than letting+ # the RUNNER answer. get_torch_index_url returns the cpu index outright for any+ # other `uname -m`, which on the ubuntu-24.04-arm leg turned these into "the+ # request selected cpu": two of them failed for a reason that is about the+ # runner, and the cpu-index CONTROLS below passed for the same wrong reason.+ 'uname() { case "${1:-}" in -m) echo x86_64 ;; *) echo Linux ;; esac; }',+ _shell_function("_rocm_torch_explicitly_requested"),+ stubs,+ # The stubs above name this host's arch family, so it is not lifted here.+ *_wheel_route_defs(arch_family = False),+ _shell_function("get_torch_index_url"),+ f"{env} get_torch_index_url",+ ]+ )+ return subprocess.run(+ ["bash", "-c", script],+ capture_output = True,+ text = True,+ env = _clean_env(),+ ).stdout+++# A bash function definition ends at its closing brace, so these are newline+# separated: two on one line is a syntax error, which the first version of this+# harness produced and read as "the flag did nothing".+_MIXED_HOST = "\n".join(+ [+ "_has_usable_nvidia_gpu() { return 0; }",+ "_has_amd_rocm_gpu() { return 0; }",+ "_probe_amd_gfx_arch() { echo gfx1201; }",+ "_amd_arch_index_family_for_gfx() { echo gfx120X-all; }",+ "_kfd_gfx_targets() { echo gfx1201; }",+ "_infer_linux_amd_gfx_arch() { echo gfx1201; }",+ "_amd_sole_index_arch() { echo gfx1201; }",+ "_amd_gpu_present_via_pci() { return 0; }",+ "_detect_rocm_version_tag() { echo rocm7.0; }",+ "_amd_agreed_index_family() { echo gfx120X-all; }",+ "_rocm_sdk_install_hint() { echo ''; }",+ "nvidia-smi() { echo 'CUDA Version: 13.0'; }",+ ]+)+++def test_the_shell_installer_selects_a_rocm_index_under_the_request(stack):+ """Fails before the fix: get_torch_index_url set _nvidia_detected=1 and returned a cu* family+ whatever the AMD probes said, so the request could not swap anything."""+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1", _MIXED_HOST)+ assert "rocm" in out or "gfx" in out, out+++def test_the_same_host_without_the_request_still_selects_cuda(stack):+ """The control, identical but for the variable."""+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=0", _MIXED_HOST)+ assert "rocm" not in out and "gfx" not in out, out+++_NO_AMD_CARD = "\n".join(+ [+ _MIXED_HOST.replace(+ "_has_amd_rocm_gpu() { return 0; }", "_has_amd_rocm_gpu() { return 1; }"+ ),+ # Every source the route test can read, silenced together. Silencing only the+ # ROCm runtime leaves the arch probes answering, which is the host the+ # runtime-less reroute exists for rather than a machine with no AMD card.+ "_probe_amd_gfx_arch() { :; }",+ "_kfd_gfx_targets() { :; }",+ "_infer_linux_amd_gfx_arch() { :; }",+ "_amd_gpu_present_via_pci() { return 1; }",+ ]+)+++def test_the_request_does_not_select_rocm_without_an_amd_card(stack):+ """A typo on a pure NVIDIA box must not turn a working CUDA machine into a ROCm or CPU one: the+ AMD route test still has to pass."""+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1", _NO_AMD_CARD)+ assert "rocm" not in out and "gfx" not in out, out+++ "rocm_gpu, archs",+ [(True, ["gfx1100"]), (False, ["gfx1151"])],+ ids = ["runtime-visible", "runtime-less-but-inferable"],+)+def test_the_cuda_repair_stands_down_under_the_request(stack, monkeypatch, rocm_gpu, archs):+ """A standalone `studio update` leaves _TORCH_BACKEND empty, so the CUDA repair runs+ first, sees an NVIDIA GPU beside the requested HIP build, reads it as poisoning and+ reinstalls the CUDA trio. The second row is the control: a runtime-less host whose arch+ is inferable is one _ensure_rocm_torch will serve, so repairing CUDA here only to have+ ROCm forced back is a reinstall cycle on every update."""+ _mixed_host(stack, monkeypatch, rocm_gpu = rocm_gpu, archs = archs, display = True)+ probed = {"ran": False}+ monkeypatch.setattr(+ stack,+ "_probe_torch_runtime",+ lambda *a, **k: (+ probed.__setitem__("ran", True),+ (True, True, "2.11.0+rocm7.0", True, False),+ )[1],+ )+ stack._ensure_cuda_torch()+ assert probed["ran"] is False+++def test_the_cuda_repair_still_runs_without_the_request(stack, monkeypatch):+ """The control: the same host, no request, and the repair must still classify."""+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ probed = {"ran": False}+ monkeypatch.setattr(+ stack,+ "_probe_torch_runtime",+ lambda *a, **k: (+ probed.__setitem__("ran", True),+ (True, True, "2.11.0+rocm7.0", True, False),+ )[1],+ )+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)+ stack._ensure_cuda_torch()+ assert probed["ran"] is True+++def test_windows_is_not_swapped_by_the_request_alone(stack, monkeypatch):+ """Windows picks the wheel in install.ps1 and republishes an expected tag that+ _ensure_expected_torch_flavor restores, so honouring the request only in Python would+ install ROCm and have it reverted."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setattr(stack, "IS_WINDOWS", True)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ detected = {"ran": False}+ monkeypatch.setattr(+ stack,+ "_detect_windows_gfx_arch",+ lambda *a, **k: (detected.__setitem__("ran", True), "gfx1151")[1],+ )+ stack._ensure_rocm_torch()+ assert detected["ran"] is False+++def _nvidia_wins(env: str, stubs: str) -> bool:+ """_nvidia_gpu_wins_over_amd() under a stubbed host."""+ script = "\n".join(+ [+ _shell_function("_rocm_torch_explicitly_requested"),+ stubs,+ *_wheel_route_defs(),+ _shell_function("_nvidia_gpu_wins_over_amd"),+ f"{env} _nvidia_gpu_wins_over_amd && echo NVIDIA || echo AMD",+ ]+ )+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, env = _clean_env())+ assert out.stdout.strip() in ("NVIDIA", "AMD"), out+ return out.stdout.strip() == "NVIDIA"+++# The probes the route test reads, not its answer: stubbing _amd_request_has_a_wheel_route+# itself would make every case below assert about the stub.+_NO_AMD = "\n".join(+ [+ "_has_amd_rocm_gpu() { return 1; }",+ "_probe_amd_gfx_arch() { :; }",+ "_kfd_gfx_targets() { :; }",+ "_infer_linux_amd_gfx_arch() { :; }",+ "_amd_gpu_present_via_pci() { return 1; }",+ ]+)++_ROUTABLE_AMD = "\n".join(+ [+ # Kept so a predicate asking the OLD presence question still RUNS: an undefined+ # function would make the unroutable case below fail for the wrong reason.+ "_has_amd_rocm_gpu() { return 0; }",+ "_probe_amd_gfx_arch() { echo gfx1201; }",+ "_kfd_gfx_targets() { echo gfx1201; }",+ "_infer_linux_amd_gfx_arch() { echo gfx1201; }",+ "_amd_gpu_present_via_pci() { return 0; }",+ ]+)++_PURE_NVIDIA = "\n".join(["_has_usable_nvidia_gpu() { return 0; }", _NO_AMD])++_PURE_AMD = "\n".join(["_has_usable_nvidia_gpu() { return 1; }", _ROUTABLE_AMD])++_MIXED = "\n".join(["_has_usable_nvidia_gpu() { return 0; }", _ROUTABLE_AMD])+++def test_the_reroute_predicate_yields_to_the_request_on_a_mixed_host():+ """The #10450 gap the selector fix alone left open: an unreadable ROCm version makes+ get_torch_index_url return the cpu index deliberately, for the per-arch reroute to rewrite."""+ assert not _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _MIXED)+++def test_the_same_mixed_host_without_the_request_keeps_cuda():+ """The control, differing only in the environment."""+ assert _nvidia_wins("", _MIXED)+++def test_the_request_does_not_yield_a_pure_nvidia_host():+ """It relaxes which vendor wins, not whether there is a card to serve."""+ assert _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _PURE_NVIDIA)+++def test_a_host_with_no_nvidia_card_is_unchanged_either_way():+ """The automatic path is untouched: with no NVIDIA GPU the answer never depended on the+ request, and must not start to."""+ assert not _nvidia_wins("", _PURE_AMD)+ assert not _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _PURE_AMD)+++def test_the_reroutes_ask_the_request_aware_predicate():+ """The predicate is only worth testing if the reroutes actually consult it."""+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"+ body = install_sh.read_text(encoding = "utf-8")+ marker = "_amd_no_rocm_version_reroute=false"+ tail = body[body.index(marker) :]+ assert "! _nvidia_gpu_wins_over_amd" in tail+ assert "! _has_usable_nvidia_gpu" not in tail, (+ "a per-arch reroute still asks the bare NVIDIA probe, so the explicit "+ "request cannot reach the AMD wheels it selected"+ )+++def test_the_cuda_repair_still_runs_when_the_request_finds_no_amd_card(stack, monkeypatch):+ """Standing down needs a card to stand down FOR, which is the shell selector's rule too: a+ request on a box with no AMD GPU selects nothing and falls through to CUDA. Leaving the+ variable set on a host whose AMD card has since been removed must not silence this repair,+ because _ensure_rocm_torch then finds no target either and a stale HIP build would be left+ on a working NVIDIA GPU with nothing to fix it."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ probed = {"ran": False}+ monkeypatch.setattr(+ stack,+ "_probe_torch_runtime",+ lambda *a, **k: (+ probed.__setitem__("ran", True),+ (True, True, "2.11.0+rocm7.0", True, False),+ )[1],+ )+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)+ stack._ensure_cuda_torch()+ assert probed["ran"] is True+++def _cuda_restore_block() -> str:+ """install.sh's request-downgrade guard, lifted by text."""+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"+ lines = install_sh.read_text(encoding = "utf-8").splitlines()+ start = next(+ (+ i+ for i, line in enumerate(lines)+ if line.startswith('if [ "$_torch_index_pinned" = false ]')+ and "_rocm_torch_explicitly_requested" in "".join(lines[i : i + 2])+ ),+ None,+ )+ if start is None:+ return ""+ end = next(i for i in range(start, len(lines)) if lines[i] == "fi")+ return "\n".join(lines[start : end + 1])+++def _index_after_the_guard(env: str, resolved: str, cuda_answer: str) -> str:+ """TORCH_INDEX_URL after the guard, given what the AMD branch resolved."""+ script = "\n".join(+ [+ _shell_function("_rocm_torch_explicitly_requested"),+ # Taken from install.sh rather than restated: a classifier restated+ # here would agree with itself, and the mirror case below is exactly+ # where it must not.+ _shell_function("_torch_index_url_leaf"),+ _shell_function("_torch_index_url_is_rocm"),+ "_has_usable_nvidia_gpu() { return 0; }",+ f"get_torch_index_url() {{ echo {cuda_answer!r}; }}",+ "_torch_index_pinned=false",+ "SKIP_TORCH=false",+ f"TORCH_INDEX_URL={resolved!r}",+ f"{env} true",+ _cuda_restore_block(),+ 'printf "%s\\n" "$TORCH_INDEX_URL"',+ ]+ )+ return _bash(script, env = _clean_env(dict([env.split("=", 1)] if "=" in env else [])))+++_CUDA = "https://download.pytorch.org/whl/cu130"+++def test_a_dead_end_amd_route_keeps_cuda():+ """ROCm 5.x, an arch no index covers, or a non-x86_64 host all end the AMD branch at the cpu+ index."""+ assert (+ _index_after_the_guard(+ "UNSLOTH_FORCE_ROCM_TORCH=1",+ resolved = "https://download.pytorch.org/whl/cpu",+ cuda_answer = _CUDA,+ )+ == _CUDA+ )+++def test_a_resolved_rocm_index_is_left_alone():+ """The control that makes the test above mean something: when the request DID reach ROCm+ wheels, the guard must not undo it."""+ rocm = "https://repo.amd.com/rocm/whl/gfx1151"+ assert (+ _index_after_the_guard(+ "UNSLOTH_FORCE_ROCM_TORCH=1",+ resolved = rocm,+ cuda_answer = _CUDA,+ )+ == rocm+ )+++def test_a_deliberate_cpu_install_without_the_request_is_untouched():+ """No request, no guard."""+ cpu = "https://download.pytorch.org/whl/cpu"+ assert _index_after_the_guard("", resolved = cpu, cuda_answer = _CUDA) == cpu+++def test_a_mirror_whose_base_path_says_rocm_is_not_a_rocm_index():+ """The whole-URL form this guard first used matched a mirror's BASE path, so+ https://mirror.local/rocm-cache/cpu read as ROCm wheels and the guard stood down -- leaving+ CPU torch on a working NVIDIA card, which is the one outcome it exists to prevent."""+ assert (+ _index_after_the_guard(+ "UNSLOTH_FORCE_ROCM_TORCH=1",+ resolved = "https://mirror.local/rocm-cache/cpu",+ cuda_answer = _CUDA,+ )+ == _CUDA+ )+++def test_a_mirrored_rocm_index_is_still_left_alone():+ """The control for the case above, one path segment apart: the same mirror serving an actual+ ROCm leaf must keep it."""+ rocm = "https://mirror.local/rocm-cache/rocm7.2"+ assert (+ _index_after_the_guard(+ "UNSLOTH_FORCE_ROCM_TORCH=1",+ resolved = rocm,+ cuda_answer = _CUDA,+ )+ == rocm+ )+++def test_the_radeon_repo_leaf_counts_as_a_rocm_index():+ """repo.radeon.com ends in rocm-rel-X.Y, which _is_pip_rocm_family_leaf declines -- correctly,+ since it answers the narrower "is this a pip ROCm FAMILY index"."""+ radeon = "https://repo.radeon.com/rocm/manylinux/rocm-rel-6.4/"+ assert (+ _index_after_the_guard(+ "UNSLOTH_FORCE_ROCM_TORCH=1",+ resolved = radeon,+ cuda_answer = _CUDA,+ )+ == radeon+ )+++def test_a_miscomputing_arch_host_keeps_the_cuda_repair(stack, monkeypatch):+ """A card is necessary and not sufficient."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: True)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ probed = {"ran": False}+ monkeypatch.setattr(+ stack,+ "_probe_torch_runtime",+ lambda *a, **k: (+ probed.__setitem__("ran", True),+ (True, True, "2.11.0+rocm7.0", True, False),+ )[1],+ )+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)+ stack._ensure_cuda_torch()+ assert probed["ran"] is True+++def test_the_rocm_installer_bails_on_the_same_arch(stack, monkeypatch):+ """The other half of the pair, and the reason the test above is not arbitrary: the helper the+ repair now consults is the one _ensure_rocm_torch itself returns on, so the two sides of the+ installer cannot disagree about whether a route exists."""+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: True)+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ installed = {"ran": False}+ monkeypatch.setattr(+ stack,+ "pip_install",+ lambda *a, **k: installed.__setitem__("ran", True),+ )+ stack._ensure_rocm_torch()+ assert installed["ran"] is False+++def _gpu_summary_branch(+ resolved: str,+ request: str,+ pinned: bool = False,+) -> str:+ """Which arm the GPU detection summary takes, given the index the resolution left."""+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"+ lines = install_sh.read_text(encoding = "utf-8").splitlines()+ head = next(+ (l for l in lines if l.startswith("if _has_usable_nvidia_gpu && ")),+ "if _nvidia_gpu_wins_over_amd; then",+ )+ # The condition spans a continuation; take the rest of it verbatim.+ if head.rstrip().endswith("\\"):+ _at = lines.index(head)+ while lines[_at].rstrip().endswith("\\"):+ _at += 1+ head = head.rstrip().rstrip("\\") + " " + lines[_at].strip()+ tail = next(+ (l for l in lines if l.startswith("elif _torch_index_url_is_rocm")),+ 'elif case "$TORCH_INDEX_URL" in */rocm*|*/gfx*) true ;; *) false ;; esac; then',+ )+ script = "\n".join(+ [+ _shell_function("_rocm_torch_explicitly_requested"),+ _shell_function("_torch_index_url_leaf"),+ _shell_function("_torch_index_url_is_rocm"),+ _shell_function("_nvidia_gpu_wins_over_amd"),+ "_has_usable_nvidia_gpu() { return 0; }",+ "_has_amd_rocm_gpu() { return 0; }",+ f"TORCH_INDEX_URL={resolved!r}",+ f"_torch_index_pinned={'true' if pinned else 'false'}",+ # export, not a VAR=VAL command prefix: a prefix applies to that one command and+ # leaves the variable unset for everything after it, so the request would never+ # be in effect and the first case below would pass without the fix.+ f"export {request}" if request else ":",+ head,+ "echo nvidia",+ tail,+ "echo amd",+ "else",+ "echo amd-cpu-fallback",+ "fi",+ ]+ )+ return _bash(script)+++def test_a_pinned_rocm_index_still_reports_the_nvidia_card():+ """A pin names a wheel family, not a card. An NVIDIA host pinned to a ROCm index has no AMD+ card to describe, so reading the resolved index alone hid the NVIDIA identity behind an+ "AMD ROCm" banner for hardware the machine does not have."""+ assert (+ _gpu_summary_branch("https://download.pytorch.org/whl/rocm7.2", "", pinned = True) == "nvidia"+ )+++def test_an_unpinned_rocm_resolution_still_reports_amd():+ """The control: the exemption is the pin, not the index. The same URL reached by resolution+ still reports the card its wheels are for."""+ assert _gpu_summary_branch("https://download.pytorch.org/whl/rocm7.2", "") == "amd"+++def test_the_summary_reports_cuda_after_the_cuda_restore():+ """The mixed host whose request found no ROCm route: the guard put the CUDA index back, but the+ request is still set and the AMD card is still there, so the reroute predicate still answers+ "AMD wins"."""+ assert _gpu_summary_branch(_CUDA, "UNSLOTH_FORCE_ROCM_TORCH=1") == "nvidia"+++def test_the_summary_still_reports_amd_when_the_request_won():+ """The control: when the request DID reach ROCm wheels, the summary must report the card those+ wheels are for, which is what the predicate was introduced to fix."""+ assert (+ _gpu_summary_branch(+ "https://repo.amd.com/rocm/whl/gfx1151",+ "UNSLOTH_FORCE_ROCM_TORCH=1",+ )+ == "amd"+ )+++def test_the_summary_on_a_pure_cuda_install_is_unchanged():+ """And the control for the control: no request, CUDA wheels, NVIDIA reported, which is what+ every automatic install on an NVIDIA host must keep seeing."""+ assert _gpu_summary_branch(_CUDA, "") == "nvidia"+++def _route_shell(probe: str, inferred: str, pci_ok: bool) -> bool:+ """install.sh's request route test, on a stubbed host."""+ script = "\n".join(+ [+ f'_probe_amd_gfx_arch() {{ printf "%s\\n" {probe!r}; }}',+ "_kfd_gfx_targets() { :; }",+ f'_infer_linux_amd_gfx_arch() {{ [ -n {inferred!r} ] && printf "%s\\n" {inferred!r}; }}',+ f"_amd_gpu_present_via_pci() {{ return {0 if pci_ok else 1}; }}",+ *_wheel_route_defs(),+ "_detect_rocm_version_tag() { printf '%s\\n' rocm7.2; }",+ "_amd_request_has_a_wheel_route && echo yes || echo no",+ ]+ )+ return _bash(script, env = _clean_env()) == "yes"+++ "probe, inferred, pci_ok, routes",+ [+ # gfx1010 (RDNA 1) is in neither the generic wheel nor any per-arch index.+ ("gfx1010", "", True, False),+ ("gfx1100", "", True, True),+ # A Strix box with no rocminfo, amd-smi or KFD node is served per-arch wheels on a+ # pure-AMD host by the reroute this predicate gates, so it is a route here too.+ ("", "gfx1151", True, True),+ ("", "gfx1030", False, False),+ # gfx906's only route is the rocm6.3 legacy tag, which opens solely when it is the+ # sole arch on the machine.+ ("gfx906", "", True, True),+ ("gfx906\ngfx1010", "", True, False),+ ],+ ids = [+ "no-index",+ "generic-wheel",+ "inferable-no-runtime",+ "declared-without-a-card",+ "gfx906-sole-arch",+ "gfx906-mixed-amd",+ ],+)+def test_the_request_needs_a_card_whose_arch_an_index_serves(probe, inferred, pci_ok, routes):+ """Presence is not the bar: an arch no index can serve must never depose a card that can,+ and each row is paired with its control."""+ assert _route_shell(probe, inferred, pci_ok = pci_ok) is routes+++def _mixed_host(stack, monkeypatch, **over):+ """The host this feature exists for, stated by difference."""+ o = dict(+ request = True,+ nvidia = True,+ rocm_gpu = True,+ archs = ["gfx1100"],+ devices = None,+ masked = None,+ kfd = [],+ wsl = False,+ display = None,+ miscomputing = False,+ machine = "x86_64",+ windows = False,+ macos = False,+ backend = "",+ inferred = None,+ rocm = (6, 4),+ torch = "2.13.0+cu130",+ declared = None,+ pin = None,+ pin_url = None,+ masks = {},+ )+ o.update(over)+ if o["display"] is None:+ o["display"] = o["rocm_gpu"]+ devices = o["archs"] if o["devices"] is None else o["devices"]++ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)+ if o["request"]:+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ for _var in (+ "UNSLOTH_ROCM_GFX_ARCH",+ "UNSLOTH_TORCH_INDEX_URL",+ "UNSLOTH_TORCH_INDEX_FAMILY",+ "UNSLOTH_ROCM_TORCH_INSTALLED",+ "HIP_VISIBLE_DEVICES",+ "ROCR_VISIBLE_DEVICES",+ "CUDA_VISIBLE_DEVICES",+ ):+ monkeypatch.delenv(_var, raising = False)+ for _var, _val in (+ ("UNSLOTH_ROCM_GFX_ARCH", o["declared"]),+ ("UNSLOTH_TORCH_INDEX_FAMILY", o["pin"]),+ ("UNSLOTH_TORCH_INDEX_URL", o["pin_url"]),+ ):+ if _val is not None:+ monkeypatch.setenv(_var, _val)+ for _var, _val in o["masks"].items():+ monkeypatch.setenv(_var, _val)++ monkeypatch.setattr(stack.platform, "machine", lambda: o["machine"])+ monkeypatch.setattr(stack, "IS_WINDOWS", o["windows"])+ monkeypatch.setattr(stack, "IS_MACOS", o["macos"])+ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "_TORCH_BACKEND", o["backend"])+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: o["nvidia"])+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: o["rocm_gpu"])+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda *a, **k: list(o["kfd"]))+ monkeypatch.setattr(stack, "_is_wsl", lambda: o["wsl"])+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: o["display"])+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: o["miscomputing"])+ monkeypatch.setattr(stack, "_infer_linux_amd_gfx_arch", lambda *a, **k: o["inferred"])+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda *a, **k: o["rocm"])+ monkeypatch.setattr(+ stack,+ "_physical_amd_gfx_archs",+ lambda: list(o["archs"]) or ([o["inferred"]] if o["inferred"] else []),+ )+ # The masked list is what a probe that HONOURS the mask returns; ignore_visible_masks asks+ # for the machine before any mask, which is the distinction the ROCr layer rests on.+ monkeypatch.setattr(+ stack,+ "_detect_amd_gfx_codes",+ lambda **k: list(+ devices if o["masked"] is None or k.get("ignore_visible_masks") else o["masked"]+ ),+ )+ monkeypatch.setattr(+ stack,+ "_probe_torch_runtime",+ lambda *a, **k: (True, True, o["torch"], "rocm" in o["torch"], ""),+ )+ # The pin readers are deliberately NOT stubbed: the environment above is already+ # cleared, so they answer None on their own, and a case that sets a pin needs the real+ # reader to see it.+ return o+++def _viable(+ stack,+ monkeypatch,+ *,+ corroborated: bool,+ archs: list,+ miscomputing = False,+ machine: str = "x86_64",+):+ _mixed_host(+ stack,+ monkeypatch,+ request = False,+ rocm_gpu = corroborated,+ archs = archs,+ miscomputing = miscomputing,+ machine = machine,+ )+ return stack._forced_rocm_route_is_viable()+++def test_a_host_with_no_published_rocm_wheels_is_not_a_viable_route(stack, monkeypatch):+ """_ensure_rocm_torch() returns on its Linux-x86_64 gate without installing anything, because+ ROCm wheels are published nowhere else."""+ assert (+ _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1100"], machine = "aarch64")+ is False+ )+++def test_the_same_host_on_x86_64_is_still_viable(stack, monkeypatch):+ """The control: same card, same corroboration, an architecture the wheels exist for."""+ assert (+ _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1100"], machine = "x86_64") is True+ )+++def test_the_python_route_test_matches_the_shell_one(stack, monkeypatch):+ """Both halves of the installer have to answer the same question the same way, or a standalone+ `studio update` undoes what install.sh chose."""+ assert _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1010"]) is False+ assert _viable(stack, monkeypatch, corroborated = True, archs = ["gfx1100"]) is True+ assert _viable(stack, monkeypatch, corroborated = False, archs = ["gfx1030"]) is False+ assert (+ _viable(+ stack,+ monkeypatch,+ corroborated = True,+ archs = ["gfx1033"],+ miscomputing = True,+ )+ is False+ )+++def test_an_unroutable_card_keeps_the_cuda_repair(stack, monkeypatch):+ """The repair had stood down for any host with a visible AMD GPU. With an unusable HIP build+ and a gfx1010 beside a working NVIDIA card, nothing then classified the stale build,+ _ensure_rocm_torch found no wheel tag, and the NVIDIA GPU was left with no working torch at+ all."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda: ["gfx1010"])+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ monkeypatch.delenv("CUDA_VISIBLE_DEVICES", raising = False)+ probed = {"ran": False}+ monkeypatch.setattr(+ stack,+ "_probe_torch_runtime",+ lambda *a, **k: (+ probed.__setitem__("ran", True),+ (True, True, "2.11.0+rocm7.0", True, False),+ )[1],+ )+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: None)+ stack._ensure_cuda_torch()+ assert probed["ran"] is True+++def test_a_declared_arch_does_not_force_rocm_over_a_working_cuda_stack(stack, monkeypatch):+ """The request skips the NVIDIA precedence return, leaving the presence gate as the only thing+ between a stale UNSLOTH_ROCM_GFX_ARCH and per-arch AMD wheels installed over CUDA on a host+ with no AMD card."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1030")+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: False)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ installed = {"ran": False}+ monkeypatch.setattr(+ stack,+ "pip_install",+ lambda *a, **k: installed.__setitem__("ran", True),+ )+ stack._ensure_rocm_torch()+ assert installed["ran"] is False+++def test_a_real_card_with_a_declared_arch_is_still_served(stack, monkeypatch):+ """The control: declaring an arch is the documented routing hint for a runtime-less host, so it+ must keep working once the silicon is corroborated."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1151")+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_explicit_unknown_family_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ monkeypatch.setattr(+ stack,+ "_probe_torch_runtime",+ lambda *a, **k: (True, True, "2.11.0", False, True),+ )+ # Not a no-op pip_install stub: this route installs through pip_install_try, which is+ # a real subprocess too, so stopping at the probe is what keeps the test hermetic.+ assert _reaches_the_version_probe(stack, monkeypatch) is True+++# A card ROCm sees perfectly well and no index can serve. The two questions disagree here,+# which is the whole point: the old predicate yields the NVIDIA GPU, the new one does not.+_UNROUTABLE_AMD = "\n".join(+ [+ "_has_usable_nvidia_gpu() { return 0; }",+ "_has_amd_rocm_gpu() { return 0; }",+ "_probe_amd_gfx_arch() { echo gfx1010; }",+ "_kfd_gfx_targets() { echo gfx1010; }",+ "_infer_linux_amd_gfx_arch() { echo gfx1010; }",+ "_amd_gpu_present_via_pci() { return 0; }",+ ]+)+++def test_the_reroute_predicate_keeps_cuda_for_an_unroutable_card():+ """The call site, not the helper: both per-arch reroutes consult this predicate, and on+ presence alone a gfx1010 cleared the way for AMD wheels that carry no kernels for it."""+ assert _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _UNROUTABLE_AMD)+++def test_the_reroute_predicate_still_yields_for_a_routable_one():+ """The control: same host, routable arch, and the request must still win."""+ assert not _nvidia_wins("UNSLOTH_FORCE_ROCM_TORCH=1", _MIXED)+++def _shell_request_flag(value: "str | None") -> bool:+ """install.sh's own request test, run verbatim, for one value of the variable."""+ # Through the environment rather than the script text: a tab quoted into a bash+ # single-quoted string arrives as a literal backslash-t, so a harness that inlined the+ # value would test a different string than the one named.+ env = dict(os.environ)+ env.pop("UNSLOTH_FORCE_ROCM_TORCH", None)+ if value is not None:+ env["UNSLOTH_FORCE_ROCM_TORCH"] = value+ script = "\n".join(+ [+ _shell_function("_rocm_torch_explicitly_requested"),+ "_rocm_torch_explicitly_requested && echo yes || echo no",+ ]+ )+ return _bash(script, env = env) == "yes"++[email protected]("value", [" true ", "\ttrue", "1 ", " ON"])+def test_the_shell_flag_is_trimmed_like_its_python_twin(value):+ """install_python_stack.py reads this variable through .strip(); install.sh did not."""+ assert _shell_request_flag(value) is True++[email protected]("value", [" ", "", "tru e", None])+def test_trimming_does_not_widen_what_counts_as_a_request(value):+ """The control."""+ assert _shell_request_flag(value) is False+++def _route_shell_masked(+ physical: "list[str]",+ rocm_tag: str = "rocm7.2",+ devices: "list[str] | None" = None,+ kfd: "list[str] | None" = None,+ **mask: str,+) -> bool:+ """The request route test on a masked host, with the mask resolution left live."""+ return (+ _bash(+ _route_script(physical, rocm_tag, devices, kfd, tail = _ROUTE_YES_NO),+ env = _route_env(mask),+ )+ == "yes"+ )+++_ROUTE_YES_NO = "_amd_request_has_a_wheel_route && echo yes || echo no"+# The published target, not the verdict: on a host whose integrated GPU enumerates first the+# interesting failure is not whether ROCm wins but which card it wins FOR.+_ROUTE_TARGET = (+ '_amd_request_has_a_wheel_route >/dev/null 2>&1; printf "%s" "${_AMD_REQUEST_TARGET_GFX:-}"'+)+++def _route_env(mask: dict) -> dict:+ return _clean_env(mask)+++def _route_target_masked(+ physical: "list[str]",+ rocm_tag: str = "rocm7.2",+ devices: "list[str] | None" = None,+ kfd: "list[str] | None" = None,+ **mask: str,+) -> str:+ """The gfx the request would install FOR, or "" when it declines."""+ return _bash(+ _route_script(physical, rocm_tag, devices, kfd, tail = _ROUTE_TARGET),+ env = _route_env(mask),+ )+++def _route_script(+ physical: "list[str]",+ rocm_tag: str,+ devices: "list[str] | None",+ kfd: "list[str] | None",+ *,+ tail: str,+) -> str:+ emit = 'printf "%s\\n" ' + " ".join(repr(a) for a in physical) if physical else ":"+ # KFD node order is what the mask ordinals index, and once rocminfo says nothing it is+ # the only ordered source left, so a test about ordering cannot leave it stubbed empty.+ kfd_emit = 'printf "%s\\n" ' + " ".join(repr(a) for a in kfd) if kfd else ":"+ script = "\n".join(+ [+ f"_probe_amd_gfx_arch() {{ {emit}; }}",+ f"_kfd_gfx_targets() {{ {kfd_emit}; }}",+ "_infer_linux_amd_gfx_arch() { :; }",+ "_amd_gpu_present_via_pci() { return 0; }",+ *_wheel_route_defs(rocminfo = _fake_rocminfo(physical if devices is None else devices)),+ f"_detect_rocm_version_tag() {{ printf '%s\\n' {rocm_tag!r}; }}",+ tail,+ ]+ )+ return script+++ "physical, mask, routes",+ [+ # No probe can answer this: rocminfo is filtered only by ROCR_VISIBLE_DEVICES and+ # amd-smi by neither, so a probe run under a HIP mask returns the whole machine and+ # the route test answered yes on the card the mask had just hidden.+ (["gfx1100", "gfx1010"], "1", False),+ (["gfx1100", "gfx1010"], "0", True),+ # Failing closed is about what cannot be resolved, not the whole variable: the+ # runtime reads left to right and stops at the first entry naming no device, so the+ # survivors are the PREFIX.+ (["gfx1100", "gfx1010"], "0,GPU-abcdef", True),+ (["gfx1100", "gfx1010"], "1,GPU-abcdef", False),+ # gfx906's only route is the rocm6.3 legacy tag, and the reroute granting it inspects+ # the UNMASKED inventory, refusing when a second AMD arch is present.+ (["gfx906", "gfx1100"], "0", False),+ # HIP remaps runtime ordinal 0 onto the head of the mask.+ (["gfx1100", "gfx1010"], "1,0", False),+ (["gfx1100", "gfx1010"], "0,1", True),+ ],+ ids = [+ "selected-unroutable",+ "selected-routable",+ "prefix-resolves",+ "prefix-unroutable",+ "gfx906-not-sole-arch",+ "mask-head-second-card",+ "mask-head-first-card",+ ],+)+def test_the_shell_route_judges_the_card_the_mask_selects(physical, mask, routes):+ """install.sh's half of the rule above, on the same host shapes."""+ assert _route_shell_masked(physical, HIP_VISIBLE_DEVICES = mask) is routes+++def test_the_flat_probes_duplicate_rows_do_not_shift_the_ordinals():+ """rocminfo names each agent's target twice, so the flat probe returns four rows for a two-card+ host."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1100", "gfx1010", "gfx1010"],+ devices = ["gfx1100", "gfx1010"],+ HIP_VISIBLE_DEVICES = "1",+ )+ is False+ )+++def test_the_same_duplicated_host_still_yields_for_the_routable_card():+ """The control: same duplicated inventory, mask pointing at the card that does have a route."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1100", "gfx1010", "gfx1010"],+ devices = ["gfx1100", "gfx1010"],+ HIP_VISIBLE_DEVICES = "0",+ )+ is True+ )+++def test_a_mask_past_the_first_device_fails_closed_with_no_per_device_list():+ """Only rocminfo enumerates in the order the masks index -- amd-smi orders by KFD discovery,+ which is why _amd_smi_hip_order exists -- so a host without it cannot say which card an+ ordinal names."""+ assert _route_shell_masked(["gfx1010", "gfx1100"], devices = [], HIP_VISIBLE_DEVICES = "1") is False+++def test_one_arch_is_still_answerable_without_a_per_device_list():+ """The control, and the reason this is not simply "no rocminfo, no route": with a single arch+ on the whole host every ordinal names it, so no enumeration order can change the answer and+ the flat inventory is enough."""+ assert _route_shell_masked(["gfx1100", "gfx1100"], devices = [], HIP_VISIBLE_DEVICES = "0") is True+++def test_discovery_order_does_not_answer_for_runtime_device_zero():+ """With no rocminfo the flat inventory came from amd-smi, which enumerates in KFD DISCOVERY+ order -- the whole reason _amd_smi_hip_order exists -- so its first row can describe a+ different GPU from the one HIP hands torch at ordinal 0. Reading it approved the swap off a+ routable first row while the gfx1010 that actually runs has kernels in no wheel."""+ assert _route_shell_masked(["gfx1100", "gfx1010"], devices = [], HIP_VISIBLE_DEVICES = "0") is False+++def test_the_kernel_topology_supplies_the_order_amd_smi_cannot():+ """KFD node order IS the order HIP and ROCr index, so the repair is to read it rather than to+ decline -- the same swap _runtime_gfx_target makes in install_python_stack.py."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1010"],+ devices = [],+ kfd = ["gfx1010", "gfx1100"],+ HIP_VISIBLE_DEVICES = "0",+ )+ is False+ )+++def test_the_same_kernel_topology_still_yields_for_the_routable_card():+ """The control: same host, same KFD order, the mask pointing at the card with a route."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1010"],+ devices = [],+ kfd = ["gfx1010", "gfx1100"],+ HIP_VISIBLE_DEVICES = "1",+ )+ is True+ )+++ "mask, routes",+ [("7", False), ("1", True)],+ ids = ["mask-resolves-nothing", "mask-names-a-device"],+)+def test_a_declared_arch_does_not_answer_over_the_mask(mask, routes):+ """UNSLOTH_ROCM_GFX_ARCH takes an early return above the mask resolution, so a declared+ arch beside HIP_VISIBLE_DEVICES=7 approved the swap where HIP exposes no device at all.+ The control keeps the escape hatch: the same declaration over a mask naming a device this+ host has must still depose CUDA."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1010"],+ devices = ["gfx1100", "gfx1010"],+ UNSLOTH_ROCM_GFX_ARCH = "gfx1100",+ HIP_VISIBLE_DEVICES = mask,+ )+ is routes+ )+++def test_the_rocr_layer_is_resolved_the_same_way():+ """rocminfo does honour this one, but the route test must not depend on which probe answered:+ amd-smi honours neither mask, so the resolution has to be its own."""+ assert _route_shell_masked(["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "1") is False+++ "mask",+ [+ {"HIP_VISIBLE_DEVICES": ""},+ {"HIP_VISIBLE_DEVICES": "-1"},+ {"ROCR_VISIBLE_DEVICES": "-1"},+ {"ROCR_VISIBLE_DEVICES": "-1", "HIP_VISIBLE_DEVICES": "0"},+ {"CUDA_VISIBLE_DEVICES": ""},+ ],+)+def test_a_mask_that_exposes_no_device_leaves_cuda_alone(mask):+ """A deliberate no-GPU selection, not a detection miss: there is nothing for the request to+ swap TO, so falling back to the physical inventory here installed ROCm wheels on a host+ whose runtime exposes no AMD GPU at all."""+ assert _route_shell_masked(["gfx1100"], **mask) is False+++def test_an_unset_mask_is_not_a_no_gpu_selection():+ """Its control, and the reason ${VAR+x} rather than ${VAR:-}: an unset mask hides nothing, and+ reading it like a set-but-empty one would keep CUDA on every host."""+ assert _route_shell_masked(["gfx1100"]) is True++[email protected]("value", ["GPU-abcdef", "9"])+def test_a_mask_this_cannot_resolve_fails_closed(value):+ """ROCr accepts UUIDs, and an ordinal can name no device at all."""+ assert _route_shell_masked(["gfx1100", "gfx1010"], HIP_VISIBLE_DEVICES = value) is False+++def test_a_masked_gfx906_alone_is_still_routable():+ """The control: on a host where gfx906 IS the sole arch, the legacy tag opens and a mask naming+ it must not change that."""+ assert _route_shell_masked(["gfx906"], HIP_VISIBLE_DEVICES = "0") is True+++def test_an_explicit_cuda_pin_outranks_the_request(stack, monkeypatch):+ """_rocm_torch_explicitly_requested's docstring promises an index pin still outranks the+ request, and _rocm_pin is the ROCm pin, so a CUDA pin leaves it None and the request skipped+ the NVIDIA-precedence return."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setenv("UNSLOTH_TORCH_INDEX_FAMILY", "cu128")+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_miscomputing_arch_host", lambda: False)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ reached = {"ran": False}+ monkeypatch.setattr(+ stack,+ "_has_rocm_gpu",+ lambda: (reached.__setitem__("ran", True), True)[1],+ )+ stack._ensure_rocm_torch()+ assert reached["ran"] is False+++def test_a_rocm_pin_is_still_honoured_over_an_nvidia_card(stack, monkeypatch):+ """The control: the pin that names ROCm wheels must still win, or the fix would read as "any+ pin keeps CUDA" and break the headless/CI case _rocm_pin exists for."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setenv("UNSLOTH_TORCH_INDEX_FAMILY", "rocm7.0")+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "")+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ # A ROCm pin skips the whole vendor-precedence block, so reaching the version probe+ # is what "the pin still won" looks like from outside.+ assert _reaches_the_version_probe(stack, monkeypatch) is True+++def _viable_masked(+ stack,+ monkeypatch,+ *,+ devices: list,+ masked: "list | None" = None,+ inferred: "str | None" = None,+ rocm: "tuple | None" = (6, 4),+ declared: "str | None" = None,+ **mask: str,+) -> bool:+ """_forced_rocm_route_is_viable on a masked host, with the resolution left live."""+ _mixed_host(+ stack,+ monkeypatch,+ request = False,+ archs = devices,+ devices = devices,+ masked = masked,+ inferred = inferred,+ rocm = rocm,+ declared = declared,+ masks = mask,+ )+ return stack._forced_rocm_route_is_viable()+++ "devices, mask, viable",+ [+ # Which CARD the mask selects, not whether some arch on the bus has a route.+ (["gfx1100", "gfx1010"], "1", False),+ (["gfx1100", "gfx1010"], "0", True),+ # _pick_visible_index folds an out-of-range ordinal and a UUID alike onto GPU 0,+ # which is right for arch SELECTION and wrong for replacing a working stack: HIP+ # exposes no device for either, so both fail closed. install.sh agrees on these.+ (["gfx1100", "gfx1010"], "7", False),+ (["gfx1100", "gfx1010"], "GPU-9d4f00a1", False),+ (["gfx1010", "gfx1100"], "1", True),+ # gfx1033 is refused outright (studio/ROCM_RDNA2_APU.md), keyed on the SELECTED+ # card: a gfx1033 anywhere would be the host-wide reading this replaces.+ (["gfx1033", "gfx1100"], "0", False),+ (["gfx1033", "gfx1100"], "1", True),+ ],+ ids = [+ "selected-unroutable",+ "selected-routable",+ "past-last-device",+ "uuid",+ "in-range-second",+ "selected-miscomputing",+ "healthy-sibling",+ ],+)+def test_the_python_route_judges_the_card_the_mask_selects(+ stack, monkeypatch, devices, mask, viable+):+ """Each row is paired with the same host masked the other way, so a fix that answered+ "never viable" fails here."""+ assert _viable_masked(stack, monkeypatch, devices = devices, HIP_VISIBLE_DEVICES = mask) is viable+++def test_a_mask_exposing_no_device_is_not_a_viable_route(stack, monkeypatch):+ """The Python half of the same rule the shell now applies: a no-GPU mask is a deliberate+ selection, so there is nothing to swap to."""+ assert (+ _viable_masked(stack, monkeypatch, devices = ["gfx1100"], HIP_VISIBLE_DEVICES = "-1") is False+ )+++def test_an_unmasked_host_is_judged_exactly_as_before(stack, monkeypatch):+ """And the control for all three: with no mask set the answer is the inventory's, which is what+ this function did before and what the shell twin still does."""+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"]) is True+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1010"]) is False+++def _rocm_repair_reached(+ stack,+ monkeypatch,+ *,+ archs: list,+ viable: bool,+ pin: "str | None" = None,+ pin_url: "str | None" = None,+) -> bool:+ """Whether _ensure_rocm_torch gets past its vendor-precedence gates on a mixed host."""+ _mixed_host(stack, monkeypatch, archs = archs, pin = pin, pin_url = pin_url)+ monkeypatch.setattr(stack, "_forced_rocm_route_is_viable", lambda: viable)+ return _reaches_the_version_probe(stack, monkeypatch)+++def test_the_request_needs_a_viable_route_to_bypass_nvidia(stack, monkeypatch):+ """Presence was the bar, and presence is not the bar this feature states."""+ assert _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1010"], viable = False) is False+++def test_a_viable_route_still_lets_the_request_through(stack, monkeypatch):+ """The control: the same mixed host with a card an index can serve."""+ assert _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1100"], viable = True) is True++[email protected]("family", ["cpu", "cu128"])+def test_an_explicit_pin_of_another_family_outranks_the_request(stack, monkeypatch, family):+ """A /cpu pin had the multi-GB ROCm stack installed here and then undone by _ensure_cpu_torch,+ on every update."""+ assert (+ _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1100"], viable = True, pin = family)+ is False+ )+++ "pin_url, reached",+ [("https://download.pytorch.org/whl/rocm6.4", True), ("https://mirror.example/simple", False)],+ ids = ["rocm-pin-agrees", "unknown-family-pin"],+)+def test_a_pin_of_another_family_is_the_only_pin_that_outranks_the_request(+ stack, monkeypatch, pin_url, reached+):+ """A pin naming ROCm agrees with the request and must not be swept up by the CPU/CUDA+ check. An unknown leaf was applied verbatim at install time, so _ensure_rocm_torch+ declines to judge it and returns before any vendor gate -- pre-existing, not added here."""+ assert (+ _rocm_repair_reached(stack, monkeypatch, archs = ["gfx1100"], viable = True, pin_url = pin_url)+ is reached+ )+++def test_the_two_mask_layers_compose_in_the_order_the_runtime_reads_them():+ """ROCr filters the physical list and renumbers the survivors; the HIP layer then indexes+ those."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "1", HIP_VISIBLE_DEVICES = "0"+ )+ is False+ )+++def test_the_same_two_layers_pointing_at_the_routable_card_still_depose_cuda():+ """The control: same host, same stacking, ROCr selecting the other card."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "0", HIP_VISIBLE_DEVICES = "0"+ )+ is True+ )+++def test_two_cards_of_one_arch_do_not_shift_the_ordinals():+ """An ordinal names a DEVICE, not an architecture."""+ assert _route_shell_masked(["gfx1010", "gfx1010", "gfx1100"], HIP_VISIBLE_DEVICES = "1") is False+++def test_the_same_host_selecting_past_the_pair_is_still_routable():+ """The control for the cardinality rule: ordinal 2 on that host really is the gfx1100, and a+ resolver that simply refused every repeated arch would fail this."""+ assert _route_shell_masked(["gfx1010", "gfx1010", "gfx1100"], HIP_VISIBLE_DEVICES = "2") is True+++def test_a_declared_arch_decides_before_the_inventory():+ """UNSLOTH_ROCM_GFX_ARCH is what _probe_amd_gfx_arch's default mode returns and what+ get_torch_index_url installs from, so judging the physical inventory instead can pass on a+ routable sibling while the wheels are chosen for the declared card."""+ assert _route_shell_masked(["gfx1100"], UNSLOTH_ROCM_GFX_ARCH = "gfx1010") is False+++def test_a_declared_arch_that_is_routable_still_deposes_cuda():+ """The control, and the #7301 host: the declaration exists to serve a runtime-less card, so a+ routable one must still win even when the inventory disagrees with it."""+ assert _route_shell_masked(["gfx1010"], UNSLOTH_ROCM_GFX_ARCH = "gfx1100") is True+++def test_a_generic_only_arch_is_judged_against_the_tag_this_host_resolves():+ """gfx950 has no per-arch index, so its only route is the generic wheel -- and the tag comes+ from the installed ROCm version, which a stale /opt/rocm beside a current amdgpu can leave+ older than the card."""+ assert _route_shell_masked(["gfx950"], rocm_tag = "rocm6.4") is False+++def test_the_same_arch_on_a_tag_that_carries_it_is_routable():+ """The control: the identical host one tag later, where the wheel does carry gfx950."""+ assert _route_shell_masked(["gfx950"], rocm_tag = "rocm7.0") is True+++def test_an_arch_with_its_own_index_is_not_held_to_the_generic_floor():+ """The boundary: gfx1200 predates the rocm6.0 generic wheel too, but it has a per-arch AMD+ index that carries it, and the reroute is what serves it."""+ assert _route_shell_masked(["gfx1200"], rocm_tag = "rocm6.0") is True+++def _corroborated_on_wsl(+ stack,+ monkeypatch,+ *,+ nvidia: bool,+ rocm_sees_a_gpu: bool = False,+) -> bool:+ """_amd_hardware_is_corroborated() on a WSL box, with the WSL evidence present."""+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: rocm_sees_a_gpu)+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: [])+ monkeypatch.setattr(stack, "_is_wsl", lambda: True)+ monkeypatch.setattr(stack, "_wsl_rocm_runtime_present", lambda: True)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: nvidia)+ return stack._amd_hardware_is_corroborated()+++def test_leftover_wsl_rocm_files_are_not_a_card_beside_an_nvidia_gpu(stack, monkeypatch):+ """/dev/dxg is the generic WSL GPU bridge an NVIDIA passthrough creates too, and librocdxg is a+ file an uninstalled ROCm leaves behind, so neither names a vendor."""+ assert _corroborated_on_wsl(stack, monkeypatch, nvidia = True) is False+++def test_the_same_wsl_box_with_no_nvidia_card_still_counts(stack, monkeypatch):+ """The control, and why this is not simply "WSL never corroborates": with no NVIDIA GPU there+ is no working stack to lose, so the leftover reading still beats declining and a WSL AMD+ host whose runtime cannot answer keeps its route."""+ assert _corroborated_on_wsl(stack, monkeypatch, nvidia = False) is True+++def test_a_wsl_box_whose_runtime_names_an_agent_counts_beside_nvidia(stack, monkeypatch):+ """The other control: a real AMD adapter under WSL is still corroborated with an NVIDIA card+ present, because the ROCm runtime enumerates it rather than a leftover file standing in for+ it."""+ assert _corroborated_on_wsl(stack, monkeypatch, nvidia = True, rocm_sees_a_gpu = True) is True+++def _needs_repair_passes_the_nvidia_gate(stack, monkeypatch, *, viable: bool) -> bool:+ """Whether _amd_torch_needs_dependency_pass() gets past its NVIDIA fast path."""++ class _Reached(Exception):+ pass++ monkeypatch.setattr(stack, "NO_TORCH", False)+ monkeypatch.setattr(stack, "IS_LINUX", True)+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(stack, "_TORCH_BACKEND", "rocm")+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_explicit_torch_index_url", lambda: None)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_rocm_torch_explicitly_requested", lambda: True)+ monkeypatch.setattr(stack, "_forced_rocm_route_is_viable", lambda: viable)++ def _sentinel():+ raise _Reached++ monkeypatch.setattr(stack, "_visible_masks_select_no_gpu", _sentinel)+ try:+ assert stack._amd_torch_needs_dependency_pass() is False+ except _Reached:+ return True+ return False+++def test_an_unroutable_request_does_not_bypass_the_nvidia_fast_path(stack, monkeypatch):+ """On a mixed host whose selected AMD card has no wheel route, _ensure_cuda_torch() and+ _ensure_rocm_torch() both leave CUDA in place."""+ assert _needs_repair_passes_the_nvidia_gate(stack, monkeypatch, viable = False) is False+++def test_a_routable_request_still_bypasses_it(stack, monkeypatch):+ """The control: the request is the whole feature, so a viable one must still outrank the NVIDIA+ card here or the repair it gates can never run."""+ assert _needs_repair_passes_the_nvidia_gate(stack, monkeypatch, viable = True) is True+++def test_an_unresolvable_mask_does_not_answer_for_the_next_host(stack, monkeypatch):+ """The flag is module state, so it is reset on entry rather than only where it is decided."""+ assert (+ _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"], HIP_VISIBLE_DEVICES = "7")+ is False+ )+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"]) is True+++def test_a_rocr_ordinal_past_the_last_device_is_not_a_viable_route(stack, monkeypatch):+ """ROCr's own filter (ROCR-Runtime, core/inc/amd_filter_device.h) surfaces the tokens that are+ "Legal and NOT Terminating", and an index terminates when it "lies outside the interval [0 -+ (numGpuDevices - 1)]" -- so ROCR_VISIBLE_DEVICES=7 on a two-GPU box surfaces nothing and the+ HIP layer above it indexes an empty list."""+ assert (+ _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "7")+ is False+ )+++def test_a_rocr_prefix_that_survives_is_still_a_viable_route(stack, monkeypatch):+ """The control, and the reason this is a prefix rule rather than "any bad token declines":+ ROCR_VISIBLE_DEVICES=0,7 terminates at the 7 and still surfaces device 0, which is routable."""+ assert (+ _viable_masked(+ stack, monkeypatch, devices = ["gfx1100", "gfx1010"], ROCR_VISIBLE_DEVICES = "0,7"+ )+ is True+ )+++def test_a_rocm_version_no_wheel_family_serves_is_not_a_viable_route(stack, monkeypatch):+ """gfx908 is in the arch tables, so the route test said yes -- but on ROCm 5.7 no generic+ rocmX.Y tag resolves, the missing-kernel reroute does not fire for an arch the generic wheel+ does carry, and _ensure_rocm_torch prints "No PyTorch wheel for ROCm 5.7" and installs+ nothing."""+ assert _viable_masked(stack, monkeypatch, devices = ["gfx908"], rocm = (5, 7)) is False+++def test_the_same_arch_on_a_version_with_a_wheel_family_is_viable(stack, monkeypatch):+ """The control: the same card on ROCm 6.4, where the tag resolves and the install proceeds."""+ assert _viable_masked(stack, monkeypatch, devices = ["gfx908"], rocm = (6, 4)) is True+++def test_an_arch_the_generic_wheel_lacks_still_routes_on_an_old_version(stack, monkeypatch):+ """The second control, and the one that keeps the version test from swallowing the repair it+ exists beside: gfx1103 has no generic kernels at any tag, so _ensure_rocm_torch reroutes it+ to AMD's per-arch index whatever the version reads."""+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1103"], rocm = (5, 7)) is True+++def test_the_route_test_does_not_read_the_hosts_rocm_version(stack, monkeypatch):+ """The guard for the pin above, since its absence is not visible in a passing run."""+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: None)+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100"]) is True+++def test_a_declared_arch_does_not_answer_over_an_unresolvable_mask_in_python(stack, monkeypatch):+ """The Python half of the shell rule above."""+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")+ assert (+ _viable_masked(stack, monkeypatch, devices = ["gfx1100", "gfx1010"], HIP_VISIBLE_DEVICES = "7")+ is False+ )+++def test_a_declared_arch_over_a_mask_that_resolves_is_still_viable_in_python(stack, monkeypatch):+ """The control, and the reason the resolution is gated on a mask being set at all: the ordinary+ declared-arch host has none, and must still route without paying for a probe."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = ["gfx1100", "gfx1010"],+ declared = "gfx1100",+ HIP_VISIBLE_DEVICES = "1",+ )+ is True+ )+++# A mixed AMD host: the arch the Deck gate refuses beside one it serves, with the kernel's+# topology naming both in the order the masks index.+_VAN_GOGH_PLUS_DGPU = "\n".join(+ [+ "_has_usable_nvidia_gpu() { return 0; }",+ "_has_amd_rocm_gpu() { return 0; }",+ "_probe_amd_gfx_arch() { printf '%s\\n' gfx1033 gfx1100; }",+ # The real map, not a constant: without it gfx1033 has no per-arch family here and+ # the route test declines for the wrong reason, so the control below would pass+ # whatever the gate does.+ '_amd_arch_index_family_for_gfx() { case "$1" in'+ " gfx1100) echo gfx110X-all ;; gfx1033) echo gfx103X-all ;; *) return 1 ;; esac; }",+ "_kfd_gfx_targets() { printf '%s\\n' gfx1033 gfx1100; }",+ "_infer_linux_amd_gfx_arch() { :; }",+ "_amd_gpu_present_via_pci() { return 0; }",+ "_detect_rocm_version_tag() { echo rocm7.0; }",+ "_amd_sole_index_arch() { :; }",+ "_amd_agreed_index_family() { :; }",+ "_rocm_sdk_install_hint() { echo ''; }",+ "nvidia-smi() { echo 'CUDA Version: 13.0'; }",+ ]+)+++def test_the_bad_arch_gate_answers_for_the_selected_card_under_the_request(stack):+ """The gate is a PRESENCE test because an unresolved host cannot say which card runs."""+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1 HIP_VISIBLE_DEVICES=1", _VAN_GOGH_PLUS_DGPU)+ assert out.strip().endswith("/rocm7.0"), out+++def test_the_same_request_selecting_the_deck_still_takes_the_cpu_index(stack):+ """The control that keeps the narrowing honest: the request cannot buy ROCm wheels for the arch+ measured to compute wrong answers."""+ out = _index_url("UNSLOTH_FORCE_ROCM_TORCH=1 HIP_VISIBLE_DEVICES=0", _VAN_GOGH_PLUS_DGPU)+ assert out.strip().endswith("/cpu"), out+++def test_the_presence_rule_is_unchanged_for_a_host_that_did_not_ask(stack):+ """And the control for every other host: with no request nothing has resolved a target, so the+ gate answers on presence exactly as before (#7776, studio/ROCM_RDNA2_APU.md)."""+ out = _index_url(+ "HIP_VISIBLE_DEVICES=1",+ _VAN_GOGH_PLUS_DGPU.replace(+ "_has_usable_nvidia_gpu() { return 0; }", "_has_usable_nvidia_gpu() { return 1; }"+ ),+ )+ assert out.strip().endswith("/cpu"), out+++def test_a_repeated_rocr_ordinal_does_not_invent_a_device(stack, monkeypatch):+ """ROCr terminates on an index that "maps to a device that has been previously selected" (ROCR-+ Runtime, core/inc/amd_filter_device.h), so ROCR_VISIBLE_DEVICES=0,0 surfaces ONE device and+ HIP_VISIBLE_DEVICES=1 above it then indexes nothing."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = ["gfx1100", "gfx1010"],+ ROCR_VISIBLE_DEVICES = "0,0",+ HIP_VISIBLE_DEVICES = "1",+ )+ is False+ )+++def test_two_distinct_rocr_ordinals_still_expose_both_devices(stack, monkeypatch):+ """The control."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = ["gfx1100", "gfx1010"],+ ROCR_VISIBLE_DEVICES = "1,0",+ HIP_VISIBLE_DEVICES = "1",+ )+ is True+ )+++def test_the_rocr_subset_is_the_prefix_of_devices_the_mask_surfaces(stack, monkeypatch):+ """The buckets behind the two cases above, since a route verdict cannot show WHICH list the HIP+ layer was handed."""+ monkeypatch.delenv("ROCR_VISIBLE_DEVICES", raising = False)+ _devices = ["gfx1100", "gfx1010", "gfx1201"]+ for _mask, _expected in (+ ("0,0", ["gfx1100"]),+ ("1,1,2", ["gfx1010"]),+ ("0,7,1", ["gfx1100"]),+ ("2,0", ["gfx1201", "gfx1100"]),+ ("7", _devices),+ ):+ monkeypatch.setenv("ROCR_VISIBLE_DEVICES", _mask)+ assert stack._rocr_visible_subset(list(_devices))[0] == _expected, _mask+++def test_a_declared_arch_resolves_its_mask_against_the_unmasked_list(stack, monkeypatch):+ """With no KFD topology (WSL) the declared-arch path falls back to rocminfo, and rocminfo is+ the one probe ROCR_VISIBLE_DEVICES renumbers."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = ["gfx1100", "gfx1010"],+ masked = ["gfx1010"],+ declared = "gfx1100",+ ROCR_VISIBLE_DEVICES = "1",+ )+ is True+ )+++def test_the_same_declared_host_with_a_mask_past_its_last_device_still_declines(stack, monkeypatch):+ """The control: the unmasked list is asked for so the ordinals can be judged, not so that every+ ordinal passes."""+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = ["gfx1100", "gfx1010"],+ masked = [],+ ROCR_VISIBLE_DEVICES = "7",+ )+ is False+ )+++def test_the_installer_stops_the_rocr_prefix_at_a_repeat_too():+ """The shell half of the duplicate rule: _amd_mask_survivors printed the repeated device a+ second time, so the HIP layer read a two-entry list and selected a phantom."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1010"],+ devices = ["gfx1100", "gfx1010"],+ ROCR_VISIBLE_DEVICES = "0,0",+ HIP_VISIBLE_DEVICES = "1",+ )+ is False+ )+++def test_the_installer_still_exposes_both_devices_for_distinct_ordinals():+ """The control, matching the Python one: ROCR_VISIBLE_DEVICES=1,0 renumbers rather than+ truncates, so HIP ordinal 1 is the routable card and the request still deposes CUDA."""+ assert (+ _route_shell_masked(+ ["gfx1100", "gfx1010"],+ devices = ["gfx1100", "gfx1010"],+ ROCR_VISIBLE_DEVICES = "1,0",+ HIP_VISIBLE_DEVICES = "1",+ )+ is True+ )+++def test_the_installer_does_not_apply_the_repeat_rule_to_the_hip_layer():+ """The rule is ROCr's, cited from its own filter; clr documents no such termination for+ HIP_VISIBLE_DEVICES, so the third argument scopes it to the layer it was read from."""+ assert (+ _route_shell_masked(+ ["gfx1010", "gfx1100"],+ devices = ["gfx1010", "gfx1100"],+ HIP_VISIBLE_DEVICES = "1,1",+ )+ is True+ )+++def test_an_unresolvable_mask_on_an_inferred_host_is_not_a_detection_miss(stack, monkeypatch):+ """Nothing enumerates a device here, so the product name is the only arch on offer and+ HIP_VISIBLE_DEVICES=1 indexes past it."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = [],+ inferred = "gfx1100",+ HIP_VISIBLE_DEVICES = "1",+ )+ is False+ )+++def test_the_same_inferred_host_without_a_mask_is_still_a_route(stack, monkeypatch):+ """The control that keeps the feature: a runtime-less but inferable AMD card is deliberately+ served per-arch wheels, so the rule must be about the mask and not about the host having no+ runtime."""+ assert _viable_masked(stack, monkeypatch, devices = [], inferred = "gfx1100") is True+++def test_the_same_inferred_host_selecting_its_only_card_is_still_a_route(stack, monkeypatch):+ """The other control, and the one that pins WHICH ordinals decline: 0 names the single card the+ name inferred, so it resolves and the route stands."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = [],+ inferred = "gfx1100",+ HIP_VISIBLE_DEVICES = "0",+ )+ is True+ )+++def test_the_installer_already_declines_that_mask():+ """The installer twin of the case above, and the reason it is a Python-only fix: with one arch+ and no per-device list, _amd_request_has_a_wheel_route resolves the mask against the single+ row and takes `[ -n "$_arwr_sel" ] || return 1`, so it has always failed closed where the+ Python half fell through to its inventory fallback."""+ assert _route_shell_masked(["gfx1100"], devices = [], HIP_VISIBLE_DEVICES = "1") is False+++def test_the_installer_still_routes_that_host_unmasked():+ """The control: the same one-arch host with no mask is exactly what the request exists to+ serve, so the decline above must belong to the mask and not to the host shape."""+ assert _route_shell_masked(["gfx1100"], devices = []) is True+++_UUID_MASK = "0,GPU-DEADBEEFDEADBEEF"+++ "devices, viable",+ [(["gfx1010", "gfx1100"], False), (["gfx1100", "gfx1100"], True)],+ ids = ["unlike-adapters", "one-architecture"],+)+def test_a_uuid_in_the_rocr_mask_is_a_rejection_only_where_it_is_ambiguous(+ stack, monkeypatch, devices, viable+):+ """ROCR_VISIBLE_DEVICES may MIX ordinals and UUIDs, and a UUID names a device this+ installer cannot place -- but where every ordinal gives the same arch there is nothing+ ambiguous about it, which is why the branch asks about unlike adapters."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = devices,+ ROCR_VISIBLE_DEVICES = _UUID_MASK,+ HIP_VISIBLE_DEVICES = "0",+ )+ is viable+ )+++def test_the_named_arch_the_message_offers_resolves_the_ambiguity(stack, monkeypatch):+ """The second control: the message names UNSLOTH_ROCM_GFX_ARCH as the way through, so setting+ it must restore the route rather than leave the host declined for good."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = ["gfx1010", "gfx1100"],+ declared = "gfx1100",+ ROCR_VISIBLE_DEVICES = _UUID_MASK,+ HIP_VISIBLE_DEVICES = "0",+ )+ is True+ )+++def test_the_ambiguity_flag_does_not_survive_the_next_host(stack, monkeypatch):+ """It is a module global read by a separate function, so a stale True would decline a later,+ unrelated host."""+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = ["gfx1010", "gfx1100"],+ ROCR_VISIBLE_DEVICES = _UUID_MASK,+ HIP_VISIBLE_DEVICES = "0",+ )+ is False+ )+ assert _viable_masked(stack, monkeypatch, devices = ["gfx1100"]) is True+++def test_an_integrated_gpu_enumerated_first_does_not_strand_the_discrete_card():+ """A Ryzen APU enumerates ahead of the discrete Radeon beside it, and the wheel family is+ picked for ONE arch."""+ assert _route_shell_masked(["gfx90c", "gfx1200"]) is True+ assert _route_target_masked(["gfx90c", "gfx1200"]) == "gfx1200"+++def test_a_routable_integrated_gpu_still_yields_to_the_discrete_one():+ """The same divergence with a quieter symptom, and the reason the fix cannot just be "decline+ when the first arch has no route": gfx1036 IS routable through gfx103X-all, so the shell+ said yes and installed for the Raphael iGPU while a gfx1100 sat beside it."""+ assert _route_target_masked(["gfx1036", "gfx1100"]) == "gfx1100"+++def test_a_discrete_card_enumerated_first_is_left_alone():+ """The control: same two cards, opposite order."""+ assert _route_target_masked(["gfx1200", "gfx90c"]) == "gfx1200"+++def test_an_integrated_gpu_alone_is_still_declined():+ """The control that keeps the fix honest."""+ assert _route_shell_masked(["gfx90c"]) is False+ assert _route_target_masked(["gfx90c"]) == ""+++def test_an_unroutable_sibling_does_not_rescue_an_unroutable_integrated_pick():+ """gfx1010 is in neither the generic wheel nor any per-arch index, so deposing gfx90c for it+ buys nothing and the request still declines."""+ assert _route_shell_masked(["gfx90c", "gfx1010"]) is False+++def test_gfx906_is_not_a_candidate_for_the_swap():+ """gfx906's only route is the rocm6.3 legacy tag, which opens solely when it is the sole arch+ on the host, so promoting it here would install a rocm7.x wheel whose BLAS has no gfx906+ kernels and strand BOTH cards."""+ assert _route_target_masked(["gfx1036", "gfx906"]) == "gfx1036"+++def test_a_pinned_host_keeps_the_device_the_user_named():+ """A set mask is the user naming a device, and _visible_devices_pinned exempts the Python rule+ for exactly that reason."""+ assert _route_target_masked(["gfx90c", "gfx1200"], HIP_VISIBLE_DEVICES = "0") == ""+ assert _route_shell_masked(["gfx90c", "gfx1200"], HIP_VISIBLE_DEVICES = "0") is False+++def _versionless_reroute_block() -> str:+ """install.sh's versionless per-arch reroute, lifted by text."""+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"+ lines = install_sh.read_text(encoding = "utf-8").splitlines()+ start = lines.index("_amd_no_rocm_version_reroute=false")+ end = next(i for i in range(start, len(lines)) if lines[i] == "esac")+ return "\n".join(lines[start : end + 1])+++def _reroute_family(physical: "list[str]", **mask: str) -> str:+ """The wheel family the versionless reroute leaves standing, or "" when it clears it."""+ emit = 'printf "%s\\n" ' + " ".join(repr(a) for a in physical)+ script = "\n".join(+ [+ "_torch_index_pinned=false",+ "SKIP_TORCH=false",+ 'TORCH_INDEX_URL="https://download.pytorch.org/whl/cpu"',+ f"_probe_amd_gfx_arch() {{ {emit}; }}",+ f"_kfd_gfx_targets() {{ {emit}; }}",+ "_infer_linux_amd_gfx_arch() { :; }",+ "_amd_gpu_present_via_pci() { return 0; }",+ "_has_amd_rocm_gpu() { return 0; }",+ # No NVIDIA card, so _nvidia_gpu_wins_over_amd answers no whatever the request+ # did and the arms below differ only in the request and the mask.+ "_has_usable_nvidia_gpu() { return 1; }",+ _shell_function("_rocm_torch_explicitly_requested"),+ *_wheel_route_defs(rocminfo = _fake_rocminfo(physical)),+ _shell_function("_nvidia_gpu_wins_over_amd"),+ _shell_function("_amd_probe_arches"),+ _shell_function("_amd_agreed_index_family"),+ _shell_function("_amd_sole_index_arch"),+ _shell_function("_hsa_spoofed_physical_gfx"),+ "_detect_rocm_version_tag() { :; }",+ _versionless_reroute_block(),+ 'printf "%s" "${_amd_probed_family:-}"',+ ]+ )+ return _bash(script, env = _route_env(mask))+++# gfx1033 shares gfx103X-all with the card beside it, which is what makes the family+# disqualification reach a routable sibling in the first place.+_DECK_PLUS_RDNA2 = ["gfx1033", "gfx1030"]+++def test_the_versionless_reroute_keeps_the_family_the_request_selected():+ """The reroute clears the family on PRESENCE of gfx1033, which is right only while the card+ that will run is unknown."""+ assert (+ _reroute_family(_DECK_PLUS_RDNA2, UNSLOTH_FORCE_ROCM_TORCH = "1", HIP_VISIBLE_DEVICES = "1")+ == "gfx103X-all"+ )+++def test_the_same_request_selecting_the_deck_still_loses_the_family():+ """The control that keeps the narrowing honest: the request cannot buy ROCm wheels for the arch+ measured to compute wrong answers (studio/ROCM_RDNA2_APU.md)."""+ assert (+ _reroute_family(_DECK_PLUS_RDNA2, UNSLOTH_FORCE_ROCM_TORCH = "1", HIP_VISIBLE_DEVICES = "0")+ == ""+ )+++def test_the_presence_rule_still_holds_for_a_host_that_did_not_ask():+ """And the control for every other host: with no request nothing resolved a target, so presence+ disqualifies the shared family as before (#7776)."""+ assert _reroute_family(_DECK_PLUS_RDNA2, HIP_VISIBLE_DEVICES = "1") == ""+++ "arch, declared, viable",+ [+ ("gfx1100", "gfx1100:sramecc-:xnack-", True),+ ("gfx1100", "gfx1100", True),+ ("gfx1010", "gfx1010:xnack-", False),+ ],+ ids = ["suffixed-routable", "bare-routable", "suffixed-unroutable"],+)+def test_a_feature_suffix_changes_the_spelling_and_not_the_answer(+ stack, monkeypatch, arch, declared, viable+):+ """rocminfo prints gcnArchName with its feature flags, and that is the spelling users copy+ into UNSLOTH_ROCM_GFX_ARCH. _amd_arch_index_url keys on the bare arch, so the suffixed+ form answered None and declined a route the plain spelling gets on the same silicon. The+ third row is the control: normalising the SPELLING must not normalise the ANSWER."""+ assert (+ _viable_masked(+ stack,+ monkeypatch,+ devices = [arch],+ inferred = declared,+ rocm = (0, 0),+ declared = declared,+ )+ is viable+ )+++# ── The feature suffix rocminfo prints, which users copy verbatim ───────────────────────+++ "declared,family",+ [+ ("gfx1100", "gfx110X-all"),+ ("gfx1100:sramecc-:xnack-", "gfx110X-all"),+ ("gfx1151", "gfx1151"),+ ("gfx1151:xnack-", "gfx1151"),+ ],+)+def test_a_feature_suffix_routes_to_the_same_index_as_the_bare_arch(+ stack, monkeypatch, declared, family+):+ """rocminfo prints gcnArchName as gfx1100:sramecc-:xnack- and that is what people paste into+ UNSLOTH_ROCM_GFX_ARCH, but both arch tables are keyed on the bare arch."""+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ url = stack._amd_arch_index_url(declared)+ assert url is not None and family in url+++def test_a_suffixed_miscomputing_arch_is_still_refused(stack, monkeypatch):+ """The control that keeps the normalisation from opening a door."""+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ assert stack._amd_arch_index_url("gfx1033") is None+ assert stack._amd_arch_index_url("gfx1033:xnack-") is None+++def test_an_unknown_arch_is_still_unrouted(stack, monkeypatch):+ """The other control: normalising must not invent a route for an arch no index carries."""+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ assert stack._amd_arch_index_url("gfx1010") is None+ assert stack._amd_arch_index_url("gfx1010:xnack-") is None+++# ── The Van Gogh gate, and what may exempt a host from it ──────────────────────────────+++def _bad_arch_verdict(+ *,+ declared: str,+ probed: str,+ request: str = "1",+) -> str:+ """Which index get_torch_index_url's miscomputing-arch gate leaves, for this host."""+ lines = (+ (Path(__file__).resolve().parents[3] / "install.sh")+ .read_text(encoding = "utf-8")+ .splitlines()+ )+ end = next(i for i, line in enumerate(lines) if "end of the miscomputing-arch gate" in line)+ # The gate opens at its own header sentence; anchoring on the first+ # `_amd_gfx_bad_arch=false` walking back would land inside the exemption branch.+ start = next(+ i+ for i in range(end, 0, -1)+ if "Archs measured to compute INCORRECTLY under ROCm" in lines[i]+ )+ # Wrapped in a function, because the lifted block ends in `return` and bash refuses+ # that at top level -- unwrapped, execution ran on past the verdict.+ gate = "_gate() {\n" + "\n".join(lines[start:end]) + "\n}"+ script = "\n".join(+ [+ _shell_function("_rocm_torch_explicitly_requested"),+ *_wheel_route_defs(arch_family = False),+ "_amd_arch_index_family_for_gfx() { case $1 in gfx1030|gfx1033) echo gfx103X-all ;;"+ " gfx1100) echo gfx110X-all ;; *) return 1 ;; esac; }",+ "_has_usable_nvidia_gpu() { return 0; }",+ "_amd_gpu_present_via_pci() { return 0; }",+ f"_probe_amd_gfx_arch() {{ echo {probed}; }}",+ f"_kfd_gfx_targets() {{ echo {probed}; }}",+ f"_infer_linux_amd_gfx_arch() {{ echo {declared or probed}; }}",+ "_detect_rocm_version_tag() { echo rocm7.1; }",+ "_ensure_rocm_probe_env() { :; }",+ '_base="https://download.pytorch.org/whl"',+ f'_amd_gfx_probe="{probed}"',+ "_amd_request_has_a_wheel_route || true",+ gate,+ # The gate either PRINTS an index and returns, or falls through printing nothing.+ "_verdict=$(_gate || true)",+ 'printf "%s\\n" "${_verdict:-kept}"',+ ]+ )+ env = _clean_env({"UNSLOTH_FORCE_ROCM_TORCH": request})+ if declared:+ env["UNSLOTH_ROCM_GFX_ARCH"] = declared+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, env = env)+ return (out.stdout.strip().splitlines() or [""])[-1]+++def test_a_declared_arch_cannot_exempt_a_physical_gfx1033():+ """A Steam Deck beside an NVIDIA card, with a stale or copied UNSLOTH_ROCM_GFX_ARCH. The+ declared gfx1030 is published as the request's target, and the gate used to read any non-+ gfx1033 target as "the selected card is fine"."""+ assert _bad_arch_verdict(declared = "gfx1030", probed = "gfx1033").endswith("/cpu")+++def test_a_probed_target_still_exempts_a_routable_sibling():+ """The control, and the case the target was published for: the probe itself resolved a gfx1100,+ so the gfx1033 in the inventory is a card the runtime did not select."""+ assert _bad_arch_verdict(declared = "", probed = "gfx1100") == "kept"+++def test_the_gate_still_fires_with_no_request_at_all():+ """The other control: presence remains the rule for every host that has not asked."""+ assert _bad_arch_verdict(declared = "", probed = "gfx1033", request = "0").endswith("/cpu")+++def _reroute_family_for_target(+ *,+ inventory: list,+ target: str,+ source: str,+ bad_arch: str = "false",+) -> str:+ """The wheel family the versionless reroute would use, for this host."""+ lines = (+ (Path(__file__).resolve().parents[3] / "install.sh")+ .read_text(encoding = "utf-8")+ .splitlines()+ )+ # From the bad-arch verdict, not from the target block alone: the gate CLEARS the family+ # before the target may set it, and a lift that starts below that clearing tests a+ # sequence install.sh never runs.+ start = next(+ i+ for i, line in enumerate(lines)+ if line.strip() == 'if [ "$_amd_reroute_bad_arch" = true ]; then'+ )+ # To the close of the OUTER if, by indentation: the block nests one, so the first+ # bare "fi" is the inner one and cutting there left an unterminated script.+ indent = len(lines[start]) - len(lines[start].lstrip())+ # Two sibling blocks at this indent: the gate's clearing, then the target's assignment.+ closes = [+ i+ for i in range(start + 1, len(lines))+ if lines[i].strip() == "fi" and len(lines[i]) - len(lines[i].lstrip()) == indent+ ]+ block = "\n".join(lines[start : closes[1] + 1])+ script = "\n".join(+ [+ _shell_function("_amd_arch_index_family_for_gfx"),+ _shell_function("_amd_probe_arches"),+ _shell_function("_amd_agreed_index_family"),+ _shell_function("_amd_sole_index_arch"),+ f'_amd_probe_out="{chr(10).join(inventory)}"',+ '_amd_probed_family=$(_amd_agreed_index_family "$_amd_probe_out") || _amd_probed_family=""',+ '_amd_probed_gfx_first=$(_amd_sole_index_arch "$_amd_probe_out") || _amd_probed_gfx_first=""',+ f'_amd_reroute_target="{target}"',+ f'_AMD_REQUEST_TARGET_SOURCE="{source}"',+ f"_amd_reroute_bad_arch={bad_arch}",+ block,+ 'printf "%s\\n" "${_amd_probed_family:-none}"',+ ]+ )+ return _bash(script)+++ "inventory, target, source, family",+ [+ (["gfx1033", "gfx1100"], "gfx1100", "probe", "gfx110X-all"),+ (["gfx1033", "gfx1100"], "gfx1100", "declared", "none"),+ (["gfx1100", "gfx1101"], "gfx1100", "probe", "gfx110X-all"),+ ],+ ids = ["cross-family-probe", "cross-family-declared", "single-family"],+)+def test_the_versionless_reroute_family_comes_from_the_selected_card(+ inventory, target, source, family+):+ """_amd_agreed_index_family needs EVERY physical AMD GPU to share a family, so a+ cross-family pair answered empty, the reroute never fired and the downgrade guard+ restored CUDA -- the request ignored on exactly the host that resolved a routable+ target. Probe-resolved only: a declared arch cannot name the card the runtime selected."""+ assert _reroute_family_for_target(inventory = inventory, target = target, source = source) == family+++def test_a_disqualified_family_is_not_put_back_by_the_target():+ """The control that matters most here."""+ assert (+ _reroute_family_for_target(+ inventory = ["gfx1033"], target = "gfx1033", source = "probe", bad_arch = "true"+ )+ == "none"+ )+++def _unreadable_version_host(stack, monkeypatch, gfx: str) -> None:+ """A mixed NVIDIA + AMD host whose ROCm version cannot be read, carrying healthy CUDA torch:+ the build the stand-down leaves in place when nothing replaces it."""+ _mixed_host(stack, monkeypatch, archs = [gfx], kfd = [gfx], rocm = None)++[email protected]("gfx", ["gfx1102", "gfx1200", "gfx1201"])+def test_an_unreadable_version_is_judged_as_the_installer_judges_it(stack, monkeypatch, gfx):+ """The two halves must not read the same unreadable version differently."""+ _unreadable_version_host(stack, monkeypatch, gfx)+ assert stack._forced_rocm_route_is_viable() is False++[email protected]("gfx", ["gfx1151", "gfx1103"])+def test_an_unreadable_version_still_serves_the_arches_that_do_install(stack, monkeypatch, gfx):+ """The narrowing control."""+ _unreadable_version_host(stack, monkeypatch, gfx)+ assert stack._forced_rocm_route_is_viable() is True++[email protected]("gfx", ["gfx1102", "gfx1200", "gfx1201"])+def test_a_readable_version_still_routes_the_same_three_arches(stack, monkeypatch, gfx):+ """The other control: the change is scoped to an unreadable version."""+ _unreadable_version_host(stack, monkeypatch, gfx)+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: (6, 4))+ assert stack._forced_rocm_route_is_viable() is True+++def _declared_on_physical(stack, monkeypatch, *, declared: str, physical: list) -> bool:+ """_forced_rocm_route_is_viable for a DECLARED arch on a known physical inventory."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", declared)+ for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(_mask, raising = False)+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda: list(physical))+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: list(physical))+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: (6, 4))+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)+ return stack._forced_rocm_route_is_viable()+++def test_a_declared_arch_cannot_exempt_a_physical_gfx1033_in_python(stack, monkeypatch):+ """A declaration is a build target the user typed, not a statement about the silicon."""+ assert (+ _declared_on_physical(+ stack, monkeypatch, declared = "gfx1030", physical = ["gfx1033", "gfx1100"]+ )+ is False+ )+++def test_a_probe_resolved_target_still_exempts_the_same_host(stack, monkeypatch):+ """The control that keeps the narrowing honest, and the case install.sh spells out by name:+ with NO declaration the probe resolves the target, the discrete-GPU preference selects the+ gfx1100 over the integrated gfx1033, and that IS a statement about the silicon."""+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(_mask, raising = False)+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)+ monkeypatch.setattr(stack, "_is_wsl", lambda: False)+ monkeypatch.setattr(stack, "_linux_amd_display_device_present", lambda: True)+ monkeypatch.setattr(stack, "_infer_linux_amd_gfx_arch", lambda: None)+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda: ["gfx1033", "gfx1100"])+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda: ["gfx1033", "gfx1100"])+ monkeypatch.setattr(+ stack, "_detect_amd_gfx_codes", lambda dedup = True, **k: ["gfx1033", "gfx1100"]+ )+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda: (6, 4))+ monkeypatch.setattr(stack, "_explicit_rocm_torch_index_url", lambda: None)+ # Proves the control is about the PROBE path: the target must be the discrete card, not+ # the integrated one the guard above rejects a declaration for.+ assert stack._runtime_gfx_target(None)[0] == "gfx1100"+ assert stack._forced_rocm_route_is_viable() is True+++def test_a_declaration_still_routes_where_no_bad_arch_is_present(stack, monkeypatch):+ """The second control: the guard keys on a MISCOMPUTING arch in the machine, not on the+ declaration disagreeing with it."""+ assert (+ _declared_on_physical(+ stack, monkeypatch, declared = "gfx1030", physical = ["gfx1030", "gfx1100"]+ )+ is True+ )+++def test_the_documented_strix_declaration_still_routes(stack, monkeypatch):+ """The third control, and the reason the guard cannot simply distrust declarations: the+ documented Strix Halo workaround declares gfx1100 on a physical gfx1151, which install.sh+ calls out by name."""+ assert (+ _declared_on_physical(stack, monkeypatch, declared = "gfx1100", physical = ["gfx1151"]) is True+ )+++# ─────────────────────────────────────────────────────────────────────────────+# The request must not authorise a swap it withdraws the moment the swap happens.+++def _viability_across_the_install(stack, monkeypatch, gfx, installed):+ """_forced_rocm_route_is_viable() on one unchanging host, asked twice: once with the CUDA wheel+ still installed, once with the ROCm wheel the request asked for in its place."""+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ monkeypatch.delenv("UNSLOTH_ROCM_GFX_ARCH", raising = False)+ for _mask in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):+ monkeypatch.delenv(_mask, raising = False)+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: True)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: True)+ monkeypatch.setattr(stack, "_kfd_gfx_targets", lambda *a, **k: [gfx])+ monkeypatch.setattr(stack, "_detect_amd_gfx_codes", lambda *a, **k: [gfx])+ monkeypatch.setattr(stack, "_physical_amd_gfx_archs", lambda *a, **k: [gfx])+ monkeypatch.setattr(stack, "_infer_linux_amd_gfx_arch", lambda *a, **k: None)+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda *a, **k: None)+ for _pin in (+ "_explicit_rocm_torch_index_url",+ "_explicit_cuda_torch_index_url",+ "_explicit_cpu_torch_index_url",+ "_explicit_torch_index_url",+ ):+ monkeypatch.setattr(stack, _pin, lambda: None)+ monkeypatch.setattr(+ stack, "_probe_torch_runtime", lambda *a, **k: (True, True, installed, "", "")+ )+ return stack._forced_rocm_route_is_viable()+++def test_the_route_is_still_viable_once_the_wheel_it_authorised_is_installed(stack, monkeypatch):+ """Measured on gfx950 with an unreadable ROCm version, the one shape where+ _rocm_torch_family_needs_repair is the only arm carrying the chain."""+ before = _viability_across_the_install(stack, monkeypatch, "gfx950", "2.9.0+cu128")+ after = _viability_across_the_install(stack, monkeypatch, "gfx950", "2.11.0+rocm7.1")+ assert before is True, "the request never authorised the swap, so there is nothing to keep"+ assert after is True, "the request withdrew the route its own install created"+++def test_an_installed_rocm_build_on_an_unroutable_card_is_still_not_a_route(stack, monkeypatch):+ """The control on the rule above: "ROCm is already installed" must not become a blanket yes."""+ assert _viability_across_the_install(stack, monkeypatch, "gfx1010", "2.11.0+rocm7.1") is False+++def test_the_cuda_repair_still_stands_down_after_the_swap(stack, monkeypatch):+ """The consequence, at the caller."""+ calls = []+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: calls.append(a[:1]))+ monkeypatch.setattr(stack, "pip_install_try", lambda *a, **k: calls.append(a[:1]))+ monkeypatch.delenv("UNSLOTH_TORCH_BACKEND", raising = False)+ monkeypatch.delenv("UNSLOTH_ROCM_TORCH_INSTALLED", raising = False)+ _viability_across_the_install(stack, monkeypatch, "gfx950", "2.11.0+rocm7.1")+ stack._ensure_cuda_torch()+ assert not calls, f"the CUDA repair overwrote the requested ROCm build: {calls}"+++# ─────────────────────────────────────────────────────────────────────────────+# A feature suffix must not change which pins the install gets.+++def _inferred_install_args(stack, monkeypatch, arch):+ """The pip arguments _ensure_rocm_torch uses on its inferred-arch branch for ``arch``."""+ calls = []+ monkeypatch.setattr(stack, "pip_install", lambda *a, **k: calls.append(a))+ monkeypatch.setattr(stack, "pip_install_try", lambda *a, **k: calls.append(a))+ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", arch)+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)+ # ROCm wheels are x86_64 only, so _ensure_rocm_torch returns on its arch gate anywhere+ # else and this helper answers [] on the ubuntu-24.04-arm runner. The case is about the+ # arch SPELLING, not the host's architecture, so the host is stated like the rest.+ monkeypatch.setattr(stack.platform, "machine", lambda: "x86_64")+ monkeypatch.setattr(stack, "IS_WINDOWS", False)+ monkeypatch.setattr(stack, "IS_MACOS", False)+ monkeypatch.setattr(stack, "_has_usable_nvidia_gpu", lambda: False)+ monkeypatch.setattr(stack, "_has_rocm_gpu", lambda: False)+ monkeypatch.setattr(+ stack, "_probe_torch_runtime", lambda *a, **k: (True, True, "2.9.0", "", "")+ )+ monkeypatch.setattr(stack, "_detect_rocm_version", lambda *a, **k: None)+ for _pin in ("_explicit_rocm_torch_index_url", "_explicit_torch_index_url"):+ monkeypatch.setattr(stack, _pin, lambda: None)+ stack._ensure_rocm_torch()+ return [a for a in (calls[0] if calls else ()) if isinstance(a, str) and a.startswith("torch")]+++ "bare,suffixed",+ [("gfx1151", "gfx1151:xnack-"), ("gfx1200", "gfx1200:sramecc+:xnack-")],+)+def test_a_suffixed_arch_installs_the_same_pins_as_the_bare_one(stack, monkeypatch, bare, suffixed):+ """gcnArchName is what rocminfo prints and what users copy into UNSLOTH_ROCM_GFX_ARCH.+ Stripping the suffix in _amd_arch_index_url is what opens the inferred-arch branch for that+ spelling; the package table two lines down was still keyed on the raw string, so it missed+ and installed unpinned torch/torchvision/torchaudio -- losing the ABI bound the table exists+ to hold, on a host the bare spelling pins correctly."""+ assert _inferred_install_args(stack, monkeypatch, suffixed) == _inferred_install_args(+ stack, monkeypatch, bare+ )+++def test_the_suffixed_arch_reaches_that_branch_at_all(stack, monkeypatch):+ """The control: the assertion above is only worth anything if the suffixed spelling actually+ installs something."""+ assert _inferred_install_args(stack, monkeypatch, "gfx1151:xnack-")++[email protected]("arch", ["gfx1102:xnack-", "gfx1030:sramecc-"])+def test_an_arch_outside_the_pin_table_is_still_bounded(stack, monkeypatch, arch):+ """The suffix strip opens this branch for archs the pin table does not name, and the fallback+ was three bare package names: those hosts reached an arch-index install with no companion+ bound at all, where every other arch-index install carries one."""+ assert _inferred_install_args(stack, monkeypatch, arch) == list(+ stack._ROCM_ARCH_INDEX_TORCH_PKG_SPEC+ )+++def test_an_arch_inside_the_pin_table_keeps_its_own_pins(stack, monkeypatch):+ """The control: the fallback must not displace the table's tighter, per-arch bound."""+ assert _inferred_install_args(stack, monkeypatch, "gfx1151") == list(+ stack._WINDOWS_ROCM_TORCH_PKG_SPECS["gfx1151"]+ )+++def _mask_probe_count(stack, monkeypatch, requested):+ """How many device probes _runtime_gfx_target runs for a DECLARED arch under a set mask."""+ probes = {"n": 0}++ def _counted(*a, **k):+ probes["n"] += 1+ return []++ monkeypatch.setenv("UNSLOTH_ROCM_GFX_ARCH", "gfx1100")+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "0")+ if requested:+ monkeypatch.setenv("UNSLOTH_FORCE_ROCM_TORCH", "1")+ else:+ monkeypatch.delenv("UNSLOTH_FORCE_ROCM_TORCH", raising = False)+ monkeypatch.setattr(stack, "_kfd_gfx_targets", _counted)+ monkeypatch.setattr(stack, "_detect_amd_gfx_codes", _counted)+ target = stack._runtime_gfx_target(None)[0]+ assert target == "gfx1100", target+ return probes["n"]+++def test_a_declared_arch_under_a_mask_costs_no_probe_unless_asked(stack, monkeypatch):+ """Those probes only fill the two mask-provenance flags, which only the forced route reads,+ and each one can spend its 15s timeout. Running them for every declared-arch host put that+ cost into an ordinary `studio update` that had asked for nothing."""+ assert _mask_probe_count(stack, monkeypatch, requested = False) == 0+++def test_the_host_that_did_ask_still_resolves_its_mask(stack, monkeypatch):+ """The control: the flags are load-bearing for the route that reads them, so the request must+ still pay for them."""+ assert _mask_probe_count(stack, monkeypatch, requested = True) > 0+++# ─────────────────────────────────────────────────────────────────────────────+# The two halves must read one mask string the same way.+++def test_an_escaped_mask_is_not_decoded_into_an_ordinal(stack):+ """`awk -v vis=...` ESCAPE-PROCESSES its operand, so a literal four-character \061 arrived+ inside awk as the ordinal 1 and the shell half selected the second card."""+ script = "\n".join(+ [_shell_function("_amd_mask_survivors"), 'printf "%s" "$(_amd_mask_survivors "$1" "$2")"']+ )+ out = subprocess.run(+ ["bash", "-c", script, "_", "gfx1010\ngfx1100", "\\061"],+ capture_output = True,+ text = True,+ env = _clean_env(),+ )+ assert out.returncode == 0, out.stderr+ assert out.stdout == "", f"an escaped ordinal resolved a device: {out.stdout!r}"+ # And the twin's answer on the same string, which is what it has to agree with.+ os.environ["HIP_VISIBLE_DEVICES"] = "\\061"+ try:+ assert stack._hip_layer_mask_names_a_device(2) is False+ finally:+ os.environ.pop("HIP_VISIBLE_DEVICES", None)+++def test_a_plain_ordinal_still_selects_its_device(stack):+ """The control: the rule above must reject the ESCAPE, not every mask."""+ script = "\n".join(+ [_shell_function("_amd_mask_survivors"), 'printf "%s" "$(_amd_mask_survivors "$1" "$2")"']+ )+ out = subprocess.run(+ ["bash", "-c", script, "_", "gfx1010\ngfx1100", "1"],+ capture_output = True,+ text = True,+ env = _clean_env(),+ )+ assert out.returncode == 0, out.stderr+ assert out.stdout == "gfx1100", out.stdout+++# ─────────────────────────────────────────────────────────────────────────────+# The miscomputing-arch gate must not inherit a target from an earlier call.+++def test_the_index_selector_does_not_inherit_a_previous_target(stack):+ """_AMD_REQUEST_TARGET_GFX is cleared by the helper that sets it, and that helper runs only+ when NVIDIA is detected AND the request is set."""+ selector = _shell_function("get_torch_index_url")+ assert '_AMD_REQUEST_TARGET_GFX=""' in selector.splitlines()[1] or any(+ line.strip() == '_AMD_REQUEST_TARGET_GFX=""' for line in selector.splitlines()[:12]+ ), "the selector must clear the published target on entry"+ assert any(+ line.strip() == '_AMD_REQUEST_TARGET_SOURCE=""' for line in selector.splitlines()[:12]+ ), "the selector must clear the target SOURCE on entry too"+++ "miscomputing, viable",+ [(True, False), (False, True)],+ ids = ["every-arch-computes-wrong", "control-healthy-host"],+)+def test_a_host_gate_decides_when_no_probe_can_name_a_device(+ stack, monkeypatch, miscomputing, viable+):+ """_miscomputing_arch_host is the only thing declining when no target resolves -- a Steam+ Deck with no rocminfo, where the narrower target refusal never runs and the fallback would+ otherwise approve the swap off the inventory."""+ _mixed_host(+ stack, monkeypatch, request = False, archs = ["gfx1033"], devices = [], miscomputing = miscomputing+ )+ assert stack._forced_rocm_route_is_viable() is viable+++ "request_set, sees_amd",+ [("", False), ("UNSLOTH_FORCE_ROCM_TORCH=1", True)],+ ids = ["default-nvidia-wins", "request-opens-the-probe"],+)+def test_the_shell_probe_hides_the_amd_card_until_it_is_asked(request_set, sees_amd):+ """install.sh's half of the gate _has_rocm_gpu carries in Python: the AMD probe returns+ early on a usable NVIDIA GPU unless this run asked for ROCm."""+ script = "\n".join(+ [+ _shell_function("_rocm_torch_explicitly_requested"),+ _shell_function("_ensure_rocm_probe_env"),+ _shell_function("_has_amd_rocm_gpu"),+ "_has_usable_nvidia_gpu() { return 0; }",+ 'rocminfo() { echo " Name: gfx1100"; }',+ f"{request_set} _has_amd_rocm_gpu && echo yes || echo no",+ ]+ )+ assert _bash(script, env = _clean_env()) == ("yes" if sees_amd else "no")diff --git a/studio/install_python_stack.py b/studio/install_python_stack.pyindex 2120b84a6f9..76830041041 100644--- a/studio/install_python_stack.py+++ b/studio/install_python_stack.py@@ -2204,6 +2204,9 @@ def _amd_arch_index_url(gfx_arch: str | None) -> str | None:mirrored/air-gapped Linux repair reaches the index install.sh chose ratherthan falling back to repo.amd.com. Both default to repo.amd.com when unset."""+ # rocminfo prints gcnArchName with its feature suffix (gfx1100:sramecc-:xnack-), which is+ # the spelling users copy into UNSLOTH_ROCM_GFX_ARCH; both tables key on the bare arch.+ gfx_arch = (gfx_arch or "").strip().split(":")[0] or Noneif IS_WINDOWS:return _windows_rocm_index_url(gfx_arch)# gfx1033 (Van Gogh) miscomputes under ROCm (studio/ROCM_RDNA2_APU.md). Without this,@@ -2498,15 +2501,32 @@ def _persist_bnb_rocm_version(version: str) -> bool:return True+def _rocm_torch_explicitly_requested() -> bool:+ """Whether UNSLOTH_FORCE_ROCM_TORCH asked for ROCm torch whatever else the host has.++ Auto-detection stops probing AMD once CUDA is usable, leaving a mixed NVIDIA+AMD host no+ route to its AMD card but an index pin, which names a wheel family rather than a+ preference (#10450). Mirrors UNSLOTH_FORCE_VULKAN; a pin still outranks it. One torch+ install serves one vendor, so this SWAPS the stack: NVIDIA is unavailable while it is set.+ """+ return (os.environ.get("UNSLOTH_FORCE_ROCM_TORCH") or "").strip().lower() in (+ "1",+ "true",+ "yes",+ "on",+ )++def _has_rocm_gpu() -> bool:"""Return True only if an actual AMD GPU is visible (not just ROCm tools installed).- Always returns False when an NVIDIA GPU is present -- NVIDIA takes- priority on mixed hosts and prevents every detection path below- (rocminfo, amd-smi, KFD sysfs) from producing a false positive even- if ROCm tools are installed alongside the NVIDIA driver.+ Returns False when an NVIDIA GPU is present -- NVIDIA takes priority on mixed+ hosts and prevents every detection path below (rocminfo, amd-smi, KFD sysfs) from+ producing a false positive even if ROCm tools are installed alongside the NVIDIA+ driver -- unless this run explicitly asked for ROCm, which is the one case where+ the AMD card is the point."""- if _has_usable_nvidia_gpu():+ if _has_usable_nvidia_gpu() and not _rocm_torch_explicitly_requested():return Falsefor cmd, check_fn in (# rocminfo: real gfx GPU ids only (gfx000 = CPU agent, "gfx11-generic" = ISA line).@@ -2628,6 +2648,16 @@ def _has_usable_nvidia_gpu() -> bool:# keeps the plain indexing behaviour._LAST_AMD_GFX_PROBE: "str | None" = None+# How the last _runtime_gfx_target call failed to resolve a target, which the target itself+# cannot carry. Only the replace-the-stack callers read these; everything else keeps the guess.+_LAST_HIP_MASK_RESOLVED = True+# One layer down: _rocr_visible_subset keeps the whole list when the first ordinal names+# nothing, so a mask exposing no device still reaches HIP as a full list.+_LAST_ROCR_MASK_RESOLVED = True+# Both masks resolved and the probes still could not say WHICH arch was selected, so "no+# target" must not be read as a detection miss.+_LAST_GFX_TARGET_AMBIGUOUS = False+def _detect_amd_gfx_codes(dedup: bool = True,@@ -2782,6 +2812,109 @@ def _gfx_route_on_host(gfx: "str | None", host_codes: "list[str] | None" = None))+def _amd_hardware_is_corroborated() -> bool:+ """AMD silicon this host can point at, with no declared arch anywhere in the chain.++ _infer_linux_amd_gfx_arch() returns UNSLOTH_ROCM_GFX_ARCH before it looks at hardware, so+ it cannot answer "is there a card". Under UNSLOTH_FORCE_ROCM_TORCH that matters: the+ request skips the NVIDIA precedence return, so a stale arch would otherwise force AMD+ wheels over a working CUDA stack on a host with no AMD GPU. Sources are the ones+ _infer_linux_amd_gfx_arch uses on its own non-declared path.+ """+ if IS_WINDOWS or IS_MACOS:+ return False+ if _has_rocm_gpu() or _kfd_gfx_targets():+ return True+ if _is_wsl():+ # WSL enumerates no PCI display device, and neither /dev/dxg (an NVIDIA passthrough creates+ # it too) nor a leftover librocdxg names a vendor, so beside a usable NVIDIA card the+ # runtime must name an agent itself.+ return _wsl_rocm_runtime_present() and not _has_usable_nvidia_gpu()+ return _linux_amd_display_device_present()+++def _forced_rocm_route_is_viable() -> bool:+ """Whether the request has something to swap TO on this host.++ The bar is _gfx_has_a_wheel_route's: an arch no index can serve must never depose a card+ that can. Presence is not it (gfx1010 is present with no route). Runtime visibility is+ not it either: a runtime-less but inferable AMD card is deliberately served per-arch+ wheels on a pure-AMD host, so requiring rocminfo would answer differently for the same+ silicon by whether an NVIDIA card sits beside it.+ """+ # ROCm wheels are x86_64 only, so _ensure_rocm_torch() installs nothing elsewhere and+ # standing the CUDA repair down leaves the venv on whatever broken torch it had.+ if platform.machine().lower() not in {"x86_64", "amd64"}:+ return False+ if not _amd_hardware_is_corroborated():+ return False+ if _miscomputing_arch_host():+ return False+ # The card the runtime will hand torch, not any sibling. _runtime_gfx_target returns the+ # whole machine beside the target, which _gfx_route_on_host's gfx906 rule needs.+ _inferred = (_infer_linux_amd_gfx_arch() or "").strip().lower().split(":")[0] or None+ _target, _, _, _host_codes = _runtime_gfx_target(_inferred)+ # A declared arch REPLACES the inventory, so a stale one hides a Van Gogh sibling+ # (studio/ROCM_RDNA2_APU.md). Probe-resolved stays exempt, as install.sh already has it.+ if (+ (os.environ.get("UNSLOTH_ROCM_GFX_ARCH") or "").strip()+ and _target is not None+ and any(_gfx in _ROCM_MISCOMPUTING_GFX for _gfx in _physical_amd_gfx_archs())+ ):+ return False+ # A mask HIP cannot resolve exposes no device, so there is nothing to swap to. Above the+ # target test, or the fallback below approves the same host off its inventory.+ if not _LAST_HIP_MASK_RESOLVED or not _LAST_ROCR_MASK_RESOLVED or _LAST_GFX_TARGET_AMBIGUOUS:+ return False+ if _target is not None:+ # Keyed on the SELECTED target: _miscomputing_arch_host above requires EVERY arch to be bad,+ # so a mask selecting the gfx1033 of a pair passed it and the install then declined.+ if _target in _ROCM_MISCOMPUTING_GFX:+ return False+ # Below ROCm 6.0 no generic tag resolves, so _ensure_rocm_torch installs nothing while+ # _ensure_cuda_torch has stood down. Ask its three version-independent arms verbatim.+ _raw_ver = _detect_rocm_version()+ _ver = _raw_ver or (0, 0)+ _declared = (os.environ.get("UNSLOTH_ROCM_GFX_ARCH") or "").strip()+ if _raw_ver is None:+ # An unreadable version is not "ROCm 0.0" to _ensure_rocm_torch: its `ver is None` branch+ # answers differently for gfx1102 / gfx1200 / gfx1201 than the (0, 0) spelling does.+ _torch_ran, _torch_imp, _torch_ver, _, _ = _probe_torch_runtime()+ _installed_ver = (_torch_ver or "").lower() if (_torch_ran and _torch_imp) else ""+ # Already on a ROCm build: the arms below ask whether _ensure_rocm_torch would install+ # SOMETHING, which after a successful swap is legitimately no, and reading that as "no+ # route" let _ensure_cuda_torch overwrite it (gfx950, unreadable ROCm version).+ if "rocm" in _installed_ver or "hip" in _installed_ver:+ return _gfx_route_on_host(_target, _host_codes or [_target])+ if (+ _explicit_rocm_torch_index_url() is None+ and not _inferred+ and not _generic_rocm_wheel_lacks_kernels(_target)+ and not _rocm_torch_family_needs_repair(_target, None, _host_codes or [_target])+ and not _rocm_compat_reroute_pending(_target, (0, 0), _installed_ver)+ ):+ return False+ return _gfx_route_on_host(_target, _host_codes or [_target])+ if (+ _explicit_rocm_torch_index_url() is None+ and _generic_pytorch_rocm_tag(_ver) is None+ and not _generic_rocm_wheel_lacks_kernels(_target, _ver)+ and not (+ _inferred+ and (_declared or not _has_rocm_gpu())+ and _amd_arch_index_url(_inferred) is not None+ )+ ):+ return False+ return _gfx_route_on_host(_target, _host_codes or [_target])+ # No target resolved: a mask exposing no GPU is deliberate, anything else is a detection+ # miss where the inventory is still the best evidence.+ if _visible_masks_select_no_gpu():+ return False+ _archs = _physical_amd_gfx_archs()+ return any(_gfx_route_on_host(_gfx, _archs) for _gfx in _archs)++def _gfx_has_a_wheel_route(gfx: "str | None") -> bool:"""Whether ANY index this installer can pick carries kernels for ``gfx``.@@ -2879,6 +3012,12 @@ def _runtime_gfx_target(matter because a runtime-only ROCm install ships neither rocminfo nor amd-smi, and withno target the callers keep a wheel with no kernels for this GPU."""+ # Reset on entry, not only where decided, so a caller never reads a previous host shape's+ # answer: every early return below leaves a mask that resolved or no list to index.+ global _LAST_HIP_MASK_RESOLVED, _LAST_ROCR_MASK_RESOLVED, _LAST_GFX_TARGET_AMBIGUOUS+ _LAST_HIP_MASK_RESOLVED = True+ _LAST_ROCR_MASK_RESOLVED = True+ _LAST_GFX_TARGET_AMBIGUOUS = False# An empty (or "-1") mask selects NO GPU, deliberately, per _visible_devices_pinned.# Decided before any probe runs, because no probe is filtered the way the reroutes need:# only ROCR_VISIBLE_DEVICES reaches rocminfo, and amd-smi and KFD sysfs are filtered by@@ -2901,6 +3040,21 @@ def _runtime_gfx_target(# handing torch a gfx1100 agent the gfx1151 wheels have no code for). Only when the# override names a DIFFERENT arch: naming the arch you spoofed TO is deliberate._spoofed = _explicit_gfx if _hsa_spoof_contradicts(_explicit_gfx) else None+ # The arch names what to BUILD for; whether the runtime exposes a device to build it for is+ # separate. Only for a host that ASKED, and only when a mask is set: the two flags below are+ # read by the forced route alone, while the probes cost up to 15s each, so collecting them+ # for every declared-arch host put a minute of timeouts into an ordinary `studio update`.+ if _rocm_torch_explicitly_requested() and _visible_devices_pinned():+ # KFD node order IS the order HIP and ROCr index, and answers on runtime-less hosts.+ # ignore_visible_masks because the ordinals below index the list BEFORE the ROCr mask.+ _mask_devices = _kfd_gfx_targets() or _detect_amd_gfx_codes(+ dedup = False, ignore_visible_masks = True+ )+ if _mask_devices:+ _LAST_ROCR_MASK_RESOLVED = _rocr_layer_mask_names_a_device(len(_mask_devices))+ _LAST_HIP_MASK_RESOLVED = _hip_layer_mask_names_a_device(+ len(_rocr_visible_subset(_mask_devices)[0])+ )return _explicit_gfx, [_explicit_gfx], _spoofed, [_explicit_gfx]gfx_devices = _detect_amd_gfx_codes(dedup = False)# Keyed to the userland probe: ROCr spoofs that reading and no other.@@ -2935,6 +3089,9 @@ def _runtime_gfx_target(f" cannot be read here, so the AMD per-gfx index is left alone.\n"f" Set UNSLOTH_ROCM_GFX_ARCH to the arch you want wheels for.\n")+ # A REJECTION, not a detection miss: without the flag a caller re-reads the arch just+ # declined and stands a working CUDA install down for a swap that is then refused.+ _LAST_HIP_MASK_RESOLVED = Falsereturn None, [], None, []gfx_devices = [inferred_linux_gfx]# The machine as the probes saw it, before the ROCr layer reduces it to a lone survivor.@@ -2982,6 +3139,8 @@ def _runtime_gfx_target(gfx_devices = _kfd_ordered_unlike_adapters = len(set(gfx_devices)) > 1_discovery_ordered = False+ # Against the list BEFORE the subset, which is what the ROCr ordinals index.+ _LAST_ROCR_MASK_RESOLVED = _rocr_layer_mask_names_a_device(len(gfx_devices))gfx_devices, _rocr_unresolved = _rocr_visible_subset(gfx_devices)# A UUID names a device this cannot place. Judged against the list BEFORE the mask# was applied: dropping the tokens that did resolve can leave one arch standing and@@ -3015,7 +3174,12 @@ def _runtime_gfx_target(f" selected cannot be read here, so the AMD per-gfx index is left alone.\n"f" Set UNSLOTH_ROCM_GFX_ARCH to the arch you want wheels for.\n")+ # A REJECTION, exactly as the mask branch above, and the message just said so.+ _LAST_GFX_TARGET_AMBIGUOUS = Truereturn None, [], None, host_codes+ # Beside the pick and against the same list, the one place both are known.+ if gfx_devices:+ _LAST_HIP_MASK_RESOLVED = _hip_layer_mask_names_a_device(len(gfx_devices))runtime_gfx = (gfx_devices[_pick_visible_index(len(gfx_devices), masks = _HIP_LAYER_MASKS)]if gfx_devices@@ -3263,6 +3427,65 @@ _HIP_LAYER_MASKS = ("HIP_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES")_INDEX_PROBE_LEN = 1 << 20+def _hip_layer_mask_names_a_device(device_count: int) -> bool:+ """Whether the HIP-layer mask, if set, names a device index this host has.++ _pick_visible_index answers 0 for a mask it cannot resolve, because for arch SELECTION a+ first-GPU guess beats no answer. Deciding whether to REPLACE a working CUDA stack is the+ opposite question: HIP exposes no device, so the ROCm wheels would land where the runtime+ hands torch nothing. install.sh fails closed on the same input.++ First-set-wins between the two spellings, as _pick_visible_index documents, against the+ list the HIP layer indexes (the ROCr survivors, which the caller has applied). True when+ no HIP-layer mask is set, which is not the same as failing to resolve one.+ """+ for _env in _HIP_LAYER_MASKS:+ _val = os.environ.get(_env)+ if _val is None:+ continue+ _val = _val.strip()+ # A no-GPU mask is deliberate, and _visible_masks_select_no_gpu already declines it.+ if _val == "" or _val == "-1":+ return False+ _first = _val.split(",")[0].strip()+ try:+ return 0 <= int(_first) < device_count+ except ValueError:+ # A UUID or junk. AMD documents UUID forms for ROCR_VISIBLE_DEVICES, a different+ # layer, resolved by _rocr_visible_subset.+ return False+ return True+++def _rocr_layer_mask_names_a_device(device_count: int) -> bool:+ """Whether ROCR_VISIBLE_DEVICES, if set, leaves the runtime at least one device.++ ROCr's filter (ROCR-Runtime, core/inc/amd_filter_device.h) keeps the tokens that are+ "Legal and NOT Terminating", and an index terminates when it "lies outside the interval+ [0 - (numGpuDevices - 1)]" or "maps to a device that has been previously selected". So+ the survivors are a PREFIX: ROCR_VISIBLE_DEVICES=7 on a two-GPU box surfaces nothing.++ _rocr_visible_subset keeps the whole list there instead, deliberately, so arch SELECTION+ still guesses GPU 0. This is the other question -- whether to REPLACE a working CUDA+ stack -- and it fails closed, as _hip_layer_mask_names_a_device does one layer up. A UUID+ resolves to no position here. install.sh composes the same rule in _amd_mask_survivors.+ """+ _raw = (os.environ.get("ROCR_VISIBLE_DEVICES") or "").strip()+ if not _raw:+ return True+ _selected: "set[int]" = set()+ for _tok in _raw.split(","):+ _tok = _tok.strip()+ try:+ _idx = int(_tok)+ except ValueError:+ break+ if not (0 <= _idx < device_count) or _idx in _selected:+ break+ _selected.add(_idx)+ return bool(_selected)++def _rocr_visible_subset(gfx_devices: "list[str]") -> "tuple[list[str], bool]":"""Apply the ROCr layer to a device list no probe filtered.@@ -3280,6 +3503,7 @@ def _rocr_visible_subset(gfx_devices: "list[str]") -> "tuple[list[str], bool]":if not _raw or not gfx_devices:return gfx_devices, False_kept: "list[str]" = []+ _seen: "set[int]" = set()_unresolved = Falsefor _tok in _raw.split(","):_tok = _tok.strip()@@ -3288,12 +3512,14 @@ def _rocr_visible_subset(gfx_devices: "list[str]") -> "tuple[list[str], bool]":except ValueError:_unresolved = True # a UUID: this names a device, but not a positioncontinue- if 0 <= _idx < len(gfx_devices):- _kept.append(gfx_devices[_idx])- # An out-of-range index keeps the whole list, deliberately: _pick_visible_index warns and- # falls back to GPU 0 for that value (matching setup.ps1's Resolve-VisibleGpuIndex), and a- # stricter rule here would split the two. ROCR_VISIBLE_DEVICES=1 on a one-GPU box is a- # typo, and reading it as "no GPU" withdraws the repair from the hosts this exists for.+ # The survivors are a PREFIX: an index terminates the mask when it falls outside the list OR+ # repeats a selection, and appending past either invents a device the runtime never exposes.+ if _idx in _seen or not (0 <= _idx < len(gfx_devices)):+ break+ _seen.add(_idx)+ _kept.append(gfx_devices[_idx])+ # A mask whose FIRST index resolves to nothing keeps the whole list, deliberately, matching+ # _pick_visible_index. _LAST_ROCR_MASK_RESOLVED is the fail-closed half.return (_kept or gfx_devices), _unresolved@@ -3913,6 +4139,15 @@ def _ensure_cuda_torch(*, probe_only: bool = False) -> "bool | None":# Never undo a deliberate ROCm install (setup.ps1 sets this marker).if os.environ.get("UNSLOTH_ROCM_TORCH_INSTALLED") == "1":return+ # Nor one this run asked for: a standalone `studio update` leaves _TORCH_BACKEND empty, so this+ # repair read the requested HIP build as poisoning and reinstalled CUDA for _ensure_rocm_torch+ # to force ROCm back (#10450). Same wheel-ROUTE predicate, so the two cannot drift.+ if (+ _rocm_torch_explicitly_requested()+ and _explicit_cuda_torch_index_url() is None+ and _forced_rocm_route_is_viable()+ ):+ return# An explicit CUDA pin commits to CUDA wheels and skips ALL GPU gates below._cuda_pinned = _explicit_cuda_torch_index_url() is not None# CUDA_VISIBLE_DEVICES="" / "-1" hides the GPU; honour it unless a CUDA index is pinned.@@ -5222,7 +5457,11 @@ def _amd_torch_needs_dependency_pass() -> bool:if _explicit_rocm_torch_index_url() is None:if _explicit_torch_index_url() is not None:return False- if _has_usable_nvidia_gpu():+ # The request outranks NVIDIA only where it has somewhere to go, or this answers True+ # forever and setup.sh reruns the dependency pass on every launch for an impossible swap.+ if _has_usable_nvidia_gpu() and not (+ _rocm_torch_explicitly_requested() and _forced_rocm_route_is_viable()+ ):return False# A hidden layer either side leaves no target to classify. Same reading the routing# guard uses, so the two can never drift.@@ -5441,6 +5680,9 @@ def _ensure_rocm_torch() -> None:if IS_WINDOWS:# An explicit ROCm pin overrides the per-arch index: retry the PINNED one, not repo.amd.com._win_rocm_pin = _explicit_rocm_torch_index_url()+ # UNSLOTH_FORCE_ROCM_TORCH is deliberately NOT read here: install.ps1 picks CUDA from the+ # NVIDIA probe and setup.ps1 republishes a tag _ensure_expected_torch_flavor() restores, so+ # honouring it only here is a multi-GB round trip. Windows needs both PowerShell installers.if _win_rocm_pin is None and _has_usable_nvidia_gpu():returngfx_arch = _detect_windows_gfx_arch()@@ -5528,13 +5770,28 @@ def _ensure_rocm_torch() -> None:_infer_linux_amd_gfx_arch() if (_rocm_pin is None and not IS_WINDOWS) else None)if _rocm_pin is None:- # NVIDIA takes precedence on mixed hosts (only if a GPU is usable).- if _has_usable_nvidia_gpu():+ # NVIDIA takes precedence unless this run asked for ROCm. The request relaxes which vendor+ # wins, not whether there is a card, so the presence test below still has to pass; a pin of+ # another known family outranks it.+ if _has_usable_nvidia_gpu() and (+ not _rocm_torch_explicitly_requested()+ or _explicit_cuda_torch_index_url() is not None+ or _explicit_cpu_torch_index_url() is not None+ ):return# _has_rocm_gpu() (rocminfo / amd-smi rows) is the authoritative AMD-host signal;# the old /opt/rocm-or-hipcc gate broke runtime-only ROCm installs.if not _has_rocm_gpu() and not _inferred_linux_gfx:return # no AMD GPU visible+ # The request skipped the NVIDIA return, and _infer_linux_amd_gfx_arch() takes+ # UNSLOTH_ROCM_GFX_ARCH before hardware, so a stale one satisfies the line above with no+ # AMD card. Same viable ROUTE bar _ensure_cuda_torch stands down on.+ if (+ _rocm_torch_explicitly_requested()+ and _has_usable_nvidia_gpu()+ and not _forced_rocm_route_is_viable()+ ):+ returnver = _detect_rocm_version()if ver is None:@@ -5605,8 +5862,17 @@ def _ensure_rocm_torch() -> None:):index_url = _amd_arch_index_url(_inferred_linux_gfx)if index_url is not None:+ # The bare arch _amd_arch_index_url resolved the URL from: stripping the suffix there is+ # what OPENS this branch for "gfx1151:xnack-", and a table keyed on the raw string then+ # misses and installs unpinned torch, losing this dict's ABI bound. _hsa_spoof_contradicts+ # likewise reads a same-card HSA override as a spoof unless asked in bare form.+ _bare_gfx = (_inferred_linux_gfx or "").strip().lower().split(":")[0]+ # Falling back UNBOUNDED was the rest of that same defect: a suffixed arch reaching this+ # branch for the first time skipped the bounds the reroute applies to every other+ # arch-index install, so the newly opened route installed a companion set nothing+ # constrained. Same tuple that path uses, so the two cannot drift._torch_pkg, _vision_pkg, _audio_pkg = _WINDOWS_ROCM_TORCH_PKG_SPECS.get(- _inferred_linux_gfx, ("torch", "torchvision", "torchaudio")+ _bare_gfx, _ROCM_ARCH_INDEX_TORCH_PKG_SPEC)_safe_print(f" {_inferred_linux_gfx} inferred (ROCm runtime not visible) -- "@@ -5631,8 +5897,8 @@ def _ensure_rocm_torch() -> None:# _inferred_linux_gfx code objects alone, and a spoof naming another arch has ROCr# hand them a device none of it matches (#7331). The install is already committed# to that arch, so declining here only guarantees it is unusable.- if _hsa_spoof_contradicts(_inferred_linux_gfx):- _clear_confirmed_hsa_spoof(_inferred_linux_gfx)+ if _hsa_spoof_contradicts(_bare_gfx):+ _clear_confirmed_hsa_spoof(_bare_gfx)# An explicit UNSLOTH_ROCM_GFX_ARCH=gfx906 pins the runtime target to the# MI50 / Radeon VII path; it must win over the Strix probe-order detectiondiff --git a/tests/sh/test_get_torch_index_url.sh b/tests/sh/test_get_torch_index_url.shindex a011cdf2083..de4e234c1c6 100755--- a/tests/sh/test_get_torch_index_url.sh+++ b/tests/sh/test_get_torch_index_url.sh@@ -33,6 +33,8 @@ _FAKE_ROCM_DIR=$(mktemp -d)# references a helper that is not extracted here, the ROCm branch hits an# undefined function, silently falls through to the CPU wheel index, and the# ROCm assertions below fail.+ sed -n '/^_rocm_torch_explicitly_requested()/,/^}/p' "$INSTALL_SH"+ echo ""sed -n '/^_ensure_rocm_probe_env()/,/^}/p' "$INSTALL_SH"echo ""sed -n '/^_probe_amd_gfx_arch()/,/^}/p' "$INSTALL_SH"diff --git a/tests/sh/test_rocm_bad_arch_gate.sh b/tests/sh/test_rocm_bad_arch_gate.shindex 3500a3c0bd4..82bb0919e3b 100755--- a/tests/sh/test_rocm_bad_arch_gate.sh+++ b/tests/sh/test_rocm_bad_arch_gate.sh@@ -146,7 +146,8 @@ trap 'rm -rf "$_FN_FILE" "$_GATE_FILE" "$_REROUTE_FILE" "$_E2E_DIR" "$_FAKE_SMI_# ROCm branch answer cpu and these pass for the wrong reason. The ROCm assertion below guards it.{for _fn in _run_bounded _cvd_hides_nvidia _has_amd_rocm_gpu _has_usable_nvidia_gpu \- _ensure_rocm_probe_env _probe_amd_gfx_arch _amd_gfx_select_ordinals \+ _ensure_rocm_probe_env _rocm_torch_explicitly_requested \+ _probe_amd_gfx_arch _amd_gfx_select_ordinals \_amd_gpu_present_via_pci \_infer_amd_gfx_arch_from_gpu_name _infer_linux_amd_gfx_arch \_amd_arch_index_family_for_gfx _trim_index_path_slashes \diff --git a/tests/sh/test_rocm_no_version_arch_route.sh b/tests/sh/test_rocm_no_version_arch_route.shindex e7fe41f3fa3..e1e87d94625 100644--- a/tests/sh/test_rocm_no_version_arch_route.sh+++ b/tests/sh/test_rocm_no_version_arch_route.sh@@ -22,6 +22,8 @@ _FAKE_PROC_NV_DIR=$(mktemp -d)echo ""sed -n '/^_has_usable_nvidia_gpu()/,/^}/p' "$INSTALL_SH"echo ""+ sed -n '/^_rocm_torch_explicitly_requested()/,/^}/p' "$INSTALL_SH"+ echo ""sed -n '/^_ensure_rocm_probe_env()/,/^}/p' "$INSTALL_SH"echo ""sed -n '/^_probe_amd_gfx_arch()/,/^}/p' "$INSTALL_SH"diff --git a/tests/sh/test_rocm_no_version_arch_route_e2e.sh b/tests/sh/test_rocm_no_version_arch_route_e2e.shindex 7bf73c99b9b..72e62261cf7 100755--- a/tests/sh/test_rocm_no_version_arch_route_e2e.sh+++ b/tests/sh/test_rocm_no_version_arch_route_e2e.sh@@ -85,7 +85,8 @@ for _fn in get_torch_index_url _has_amd_rocm_gpu _has_usable_nvidia_gpu \_infer_amd_gfx_arch_from_gpu_name _amd_gpu_present_via_pci _run_bounded \_rocm_tag_from_amd_smi _rocm_tag_from_hipconfig _rocm_tag_from_rpm \_rocm_tag_from_dpkg _rocm_tag_from_version_file _highest_rocm_tag \- _trim_index_path_slashes _cvd_hides_nvidia _ensure_rocm_probe_env; do+ _trim_index_path_slashes _cvd_hides_nvidia _ensure_rocm_probe_env \+ _rocm_torch_explicitly_requested; dogrep -q "^$_fn()" "$_FUNCS" || \_fatal "install.sh no longer defines $_fn() at column 0 (splice would silently answer cpu)"donediff --git a/tests/sh/test_rocm_version_source_disagreement.sh b/tests/sh/test_rocm_version_source_disagreement.shindex c3c500a98b5..0d04e394309 100755--- a/tests/sh/test_rocm_version_source_disagreement.sh+++ b/tests/sh/test_rocm_version_source_disagreement.sh@@ -35,6 +35,8 @@ _FAKE_PROC_NV_DIR=$(mktemp -d)echo ""sed -n '/^_has_usable_nvidia_gpu()/,/^}/p' "$INSTALL_SH"echo ""+ sed -n '/^_rocm_torch_explicitly_requested()/,/^}/p' "$INSTALL_SH"+ echo ""sed -n '/^_ensure_rocm_probe_env()/,/^}/p' "$INSTALL_SH"echo ""sed -n '/^_probe_amd_gfx_arch()/,/^}/p' "$INSTALL_SH"diff --git a/tests/studio/install/test_rocm_arch_table_parity.py b/tests/studio/install/test_rocm_arch_table_parity.pyindex b595add0e5a..bb303cfbae0 100644--- a/tests/studio/install/test_rocm_arch_table_parity.py+++ b/tests/studio/install/test_rocm_arch_table_parity.py@@ -712,10 +712,12 @@ class TestTorch211PinAllowlistParity:class TestShadowingIntegratedGfxParity:- """The shadowing-APU skip (#7776) exists twice: studio/setup.ps1 resolves the- arch and builds $ROCmIndexUrl before it ever invokes the Python stack- installer, so both copies of the list have to agree or one entry point keeps- installing the iGPU's wheel family."""+ """The shadowing-APU skip (#7776) exists four times now: studio/setup.ps1+ resolves the arch and builds $ROCmIndexUrl before it ever invokes the Python+ stack installer, install_llama_prebuilt.py honours setup's repick, and+ install.sh answers the ROCm-request route on Linux before the Python stack+ runs at all. Every copy has to agree or one entry point keeps installing the+ iGPU's wheel family."""_STRIX = {"gfx1150", "gfx1151", "gfx1152"}@@ -734,6 +736,20 @@ class TestShadowingIntegratedGfxParity:return set(ast.literal_eval(node.value.args[0]))raise AssertionError("SHADOWING_INTEGRATED_GFX not found in install_llama_prebuilt.py")+ def _install_sh_list(self):+ body = _sh_function_body(+ _INSTALL_SH.read_text(encoding = "utf-8"), "_amd_gfx_is_shadowing_integrated"+ )+ m = re.search(r"\n\s*(gfx[^)]*)\)\s*return 0", body)+ assert m, "the shadowing-APU arm was not found in _amd_gfx_is_shadowing_integrated"+ return {arch.strip() for arch in m.group(1).split("|")}++ def test_install_sh_matches_install_python_stack(self):+ # install.sh exports the family it chose and _ensure_rocm_torch returns on its first+ # line for a non-ROCm one, so the shell picking the iGPU here is final and the Python+ # preference never runs to correct it.+ assert self._install_sh_list() == set(stack_mod._SHADOWING_INTEGRATED_GFX)+def test_setup_ps1_matches_install_python_stack(self):assert self._setup_ps1_list() == set(stack_mod._SHADOWING_INTEGRATED_GFX)@@ -748,6 +764,7 @@ class TestShadowingIntegratedGfxParity:assert not (self._STRIX & set(stack_mod._SHADOWING_INTEGRATED_GFX))assert not (self._STRIX & self._setup_ps1_list())assert not (self._STRIX & self._prebuilt_list())+ assert not (self._STRIX & self._install_sh_list())if __name__ == "__main__":diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.pyindex e9a76b0d3f0..a1dab177fa9 100644--- a/tests/studio/install/test_rocm_support.py+++ b/tests/studio/install/test_rocm_support.py@@ -3013,7 +3013,14 @@ class TestInstallShStructure:)def test_cuda_precedence(self):- """ROCm detection runs only when NVIDIA is absent (check runtime ordering in get_torch_index_url)."""+ """ROCm detection runs only when NVIDIA is absent, or when ROCm was ASKED for.++ The automatic profile still gives CUDA precedence: that is the guarantee, and a+ false AMD positive would swap a working install. The one exception is an+ explicit UNSLOTH_FORCE_ROCM_TORCH request, which is the only route a mixed+ NVIDIA+AMD host has to its AMD card (#10450), so any AMD probe reached before+ the no-NVIDIA branch has to be guarded by that request and nothing else.+ """sh_path = PACKAGE_ROOT / "install.sh"source = sh_path.read_text(encoding = "utf-8")body = _extract_sh_function_body(source, "get_torch_index_url")@@ -3023,16 +3030,43 @@ class TestInstallShStructure:no_nvidia_branch = body.find('if [ "$_nvidia_detected" -eq 0 ]')if no_nvidia_branch < 0:no_nvidia_branch = body.find('if [ -z "$_smi" ]')- rocm_call = body.find("_has_amd_rocm_gpu")assert nvidia_call >= 0, "get_torch_index_url should call _has_usable_nvidia_gpu"assert no_nvidia_branch >= 0, "get_torch_index_url should gate ROCm on no-nvidia branch"- assert (- rocm_call > no_nvidia_branch- ), "ROCm detection should sit inside the 'no NVIDIA' branch"assert (nvidia_call < no_nvidia_branch), "NVIDIA detection should run before the no-NVIDIA branch"+ # The automatic path is unchanged: an AMD probe still sits inside the branch.+ assert (+ body.find("_has_amd_rocm_gpu", no_nvidia_branch) > no_nvidia_branch+ ), "ROCm detection should sit inside the 'no NVIDIA' branch"++ # Anything earlier has to be the explicit request, judged on the shell statement+ # it belongs to rather than on the whole file: a bare probe before the branch+ # would give AMD precedence over CUDA on every automatic install.+ #+ # Comment lines are dropped first, since the comment explaining the guard names+ # the probe it guards, and continuation lines are then joined, since the guard+ # and the probe sit either side of a backslash.+ lines = [+ line+ for line in body[:no_nvidia_branch].splitlines()+ if not line.lstrip().startswith("#")+ ]+ statement, guarded = [], []+ for line in lines:+ statement.append(line)+ if not line.rstrip().endswith("\\"):+ guarded.append(" ".join(statement))+ statement = []+ for stmt in guarded:+ if "_has_amd_rocm_gpu" not in stmt:+ continue+ assert "_rocm_torch_explicitly_requested" in stmt, (+ "an AMD probe before the no-NVIDIA branch must be guarded by "+ f"_rocm_torch_explicitly_requested, got: {stmt.strip()!r}"+ )+def test_bitsandbytes_amd_install(self):"""install.sh should install bitsandbytes for AMD when ROCm detected."""sh_path = PACKAGE_ROOT / "install.sh"