Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Name an AMD device node this account cannot open, instead of reading it as no GPU (#10473)

Daniel Hand367996767f3
diff --git a/install.sh b/install.sh
index 965311f02af..e9d3978e93e 100755
--- a/install.sh
+++ b/install.sh
@@ -3719,9 +3719,15 @@ _ensure_rocm_probe_env() {
fi
}
+# Whether ROCm can see an AMD GPU. A usable NVIDIA card wins by default; a diagnosis that
+# has already established the run opens AMD nodes passes "ignore-nvidia" to skip that.
+# The veto is inside this function rather than in a wrapper, because every tests/sh harness
+# lifts probes one function at a time by name (sed -n '/^_name()/,/^}/p'), and a wrapper
+# whose helper is not also lifted calls nothing: the ROCm branch would then fall silently
+# through to the CPU wheel index.
_has_amd_rocm_gpu() {
_ensure_rocm_probe_env
- if _has_usable_nvidia_gpu; then
+ if [ "${1:-}" != "ignore-nvidia" ] && _has_usable_nvidia_gpu; then
return 1
fi
if command -v rocminfo >/dev/null 2>&1 && \
@@ -3754,6 +3760,305 @@ _amd_gpu_present_via_pci() {
return 1
}
+# The nodes the enumeration below looks at. Extracted so a test can name its own set:
+# the paths are absolute, so a harness cannot otherwise reach this rule at all.
+_amd_candidate_nodes() {
+ printf '%s\n' /dev/kfd /dev/dri/renderD*
+}
+
+# The PCI vendor a render node reports, or a non-zero exit when sysfs will not say.
+# Mirrors utils/hardware/amd.py::_render_node_vendor, including the distinction that
+# matters: unreadable is not the same answer as "not AMD".
+_amd_render_node_vendor() {
+ _arnv_file="/sys/class/drm/${1##*/}/device/vendor"
+ [ -r "$_arnv_file" ] || return 1
+ read -r _arnv_vendor < "$_arnv_file" 2>/dev/null || return 1
+ printf '%s' "$_arnv_vendor"
+}
+
+# Prints the AMD device nodes that exist but this user cannot open, one per line. On a
+# stock distribution /dev/kfd and /dev/dri/renderD* are root:render mode 0660, so an
+# account outside that group passes every -e test and then cannot open the device: HIP
+# counts no devices, the Vulkan loader enumerates none, and the install looks like a host
+# with no GPU (#10466). -r and -w, matching what the runtimes need; root prints nothing.
+# AMD-owned nodes only: render nodes are root:render for EVERY vendor, so an NVIDIA-only
+# box has the same closed list and none of the problem, and the group advice would be wrong
+# there. sysfs is world-readable, so ownership is answered without testing the access.
+_amd_nodes_closed_to_this_user() {
+ _anctu_amd_in_topology=""
+ _amd_candidate_nodes | while IFS= read -r _node; do
+ [ -e "$_node" ] || continue
+ { [ -r "$_node" ] && [ -w "$_node" ]; } && continue
+ if [ "$_node" = /dev/kfd ]; then
+ # vendor_id 4098 = 0x1002, the same AMD guard _has_amd_rocm_gpu uses: NVIDIA's
+ # open kernel module registers KFD nodes of its own. A topology that could not
+ # be READ is not one that named another vendor, and only the second is evidence,
+ # so where it is unknown DRM confirms the same silicon independently. Dropping
+ # the node there left the installer naming only the render node's group, which
+ # where the two differ (video against render) leaves KFD shut.
+ # Mirrors utils/hardware/amd.py::amd_nodes_closed_to_this_user.
+ _anctu_kfd_state=0
+ _kfd_topology_amd_state || _anctu_kfd_state=$?
+ if [ "$_anctu_kfd_state" -eq 1 ]; then
+ continue
+ elif [ "$_anctu_kfd_state" -ne 0 ]; then
+ _a_confirmed_amd_render_node_exists || continue
+ fi
+ elif _node_vendor=$(_amd_render_node_vendor "$_node"); then
+ [ "$_node_vendor" = "0x1002" ] || continue
+ else
+ # A vendor sysfs will not name is not a vendor that is not AMD, and a
+ # container mapping /dev/dri while hiding those attributes is the shape this
+ # diagnosis exists for: dropping the node printed no repair at all, while
+ # _amd_render_node_present reads the same unknown as PRESENT and withdraws the
+ # missing-node sentence. The world-readable KFD topology answers instead.
+ # Mirrors utils/hardware/amd.py::amd_nodes_closed_to_this_user.
+ if [ -z "$_anctu_amd_in_topology" ]; then
+ if _kfd_topology_has_an_amd_gpu; then
+ _anctu_amd_in_topology=yes
+ else
+ _anctu_amd_in_topology=no
+ fi
+ fi
+ [ "$_anctu_amd_in_topology" = yes ] || continue
+ fi
+ printf '%s\n' "$_node"
+ done
+}
+
+# Whether SOME AMD render node on this host is open to this account: the mirror of the
+# enumeration above, same vendor guard. A node this account can already open means the
+# closed ones do not stop every GPU backend on the box, only the card behind them.
+# Mirrors utils/hardware/amd.py::an_amd_render_node_is_open.
+_an_amd_render_node_is_open() {
+ for _anro_node in /dev/dri/renderD*; do
+ [ -e "$_anro_node" ] || continue
+ _anro_vendor_file="/sys/class/drm/${_anro_node##*/}/device/vendor"
+ [ -r "$_anro_vendor_file" ] || continue
+ read -r _anro_vendor < "$_anro_vendor_file" 2>/dev/null || continue
+ [ "$_anro_vendor" = "0x1002" ] || continue
+ { [ -r "$_anro_node" ] && [ -w "$_anro_node" ]; } && return 0
+ done
+ return 1
+}
+
+# Whether KFD enumerates an AMD GPU node, the AMD-presence signal that survives a host with
+# no render node at all. vendor_id 4098 = 0x1002; the KFD CPU node reports 0 and NVIDIA's
+# open kernel module registers 4318. Exit 0 = the topology names an AMD GPU, 1 = it was READ
+# and names none, 2 = it could not be read at all. The third is not the second: a container
+# can map /dev/kfd while hiding /sys/class/kfd, and collapsing the two drops the node from
+# the closed list on exactly that host.
+# Mirrors utils/hardware/amd.py::_kfd_topology_has_an_amd_gpu and _kfd_topology_amd_state.
+_kfd_topology_amd_state() {
+ _ktas=$(awk '
+ FNR == 1 { read_one = 1 }
+ /vendor_id/ && $2 == 4098 { found = 1 }
+ END { print (read_one + 0) ":" (found + 0) }
+ ' /sys/class/kfd/kfd/topology/nodes/*/properties 2>/dev/null) || _ktas=""
+ case "$_ktas" in
+ *:1) return 0 ;;
+ 1:0) return 1 ;;
+ *) return 2 ;;
+ esac
+}
+
+# Whether the /dev/kfd that EXISTS here is one the AMD driver put there: the branch below
+# claims the AMD kernel stack is loaded, and a node merely existing does not establish that.
+# A tri-state, not the stricter "require vendor 4098": state 1 (read, no AMD agent) is the
+# only reading that CONTRADICTS the claim, while state 2 (unreadable -- the container that
+# masks /sys/class/kfd) leaves it unproven, so that host keeps the diagnosis it has today.
+_kfd_node_is_amds() {
+ _kina_state=0
+ _kfd_topology_amd_state || _kina_state=$?
+ [ "$_kina_state" -ne 1 ]
+}
+
+# Whether this host has AMD silicon behind a /dev/kfd that is not there. Mirrors
+# utils/hardware/amd.py::_amd_nodes_the_runtime_lacks question for question; the two halves
+# must agree or one stays silent on a host the other diagnoses.
+#
+# The topology settles it whenever it can be READ. Where it cannot (state 2: --device
+# /dev/dri with /sys/class/kfd masked) a CONFIRMED AMD render node stands in, because that
+# host had no diagnosis at all -- amd-smi answers through libdrm and silences the PCI
+# branches, and a node that does not EXIST never reaches the closed-node list. Confirmed,
+# not merely present, or an NVIDIA-only box masked the same way would claim an AMD card.
+_amd_silicon_behind_a_missing_kfd() {
+ if _kfd_topology_has_an_amd_gpu; then return 0; fi
+ # Captured rather than tested inline: this runs under set -e, where a bare non-zero
+ # command ends the installer instead of answering the question.
+ _asbmk_state=0
+ _kfd_topology_amd_state || _asbmk_state=$?
+ [ "$_asbmk_state" -eq 2 ] || return 1
+ _a_confirmed_amd_render_node_exists
+}
+
+# Whether DRM names an AMD render node outright, with the vendor actually READ. The strict
+# counterpart to _amd_render_node_present, which counts an unreadable vendor as present on
+# purpose: this is used as INDEPENDENT evidence of AMD silicon, so an unknown vendor would
+# let an NVIDIA-only host claim one. Mirrors
+# utils/hardware/amd.py::_a_confirmed_amd_render_node_exists.
+_a_confirmed_amd_render_node_exists() {
+ for _acarne_node in /dev/dri/renderD*; do
+ [ -e "$_acarne_node" ] || continue
+ _acarne_vendor=$(_amd_render_node_vendor "$_acarne_node") || continue
+ [ "$_acarne_vendor" = "0x1002" ] && return 0
+ done
+ return 1
+}
+
+_kfd_topology_has_an_amd_gpu() {
+ _kthag=0
+ _kfd_topology_amd_state || _kthag=$?
+ [ "$_kthag" -eq 0 ]
+}
+
+# Whether any AMD render node is PRESENT, whatever this account can do with it: ROCr opens
+# one to reach amdgpu, so a container given --device /dev/kfd alone still cannot start ROCm
+# and no group creates the node (docker/run.sh passes both devices for that reason).
+# A vendor that cannot be READ is not a vendor that is absent -- calling it absence sent
+# the user to recreate a container with the device it already has -- so unknown counts as
+# present, keeping the closed-node diagnosis reachable. Mirrors
+# utils/hardware/amd.py::_amd_render_node_exists.
+_amd_render_node_present() {
+ _arnp_unknown=false
+ for _arnp_node in /dev/dri/renderD*; do
+ [ -e "$_arnp_node" ] || continue
+ _arnp_vendor_file="/sys/class/drm/${_arnp_node##*/}/device/vendor"
+ if [ -r "$_arnp_vendor_file" ] && \
+ read -r _arnp_vendor < "$_arnp_vendor_file" 2>/dev/null; then
+ [ "$_arnp_vendor" = "0x1002" ] && return 0
+ else
+ _arnp_unknown=true
+ fi
+ done
+ [ "$_arnp_unknown" = true ]
+}
+
+# A value as a single shell word, for a command the user is going to paste. NSS names are
+# not identifiers -- winbind hands back DOMAIN\user, and a group name may carry whitespace
+# or a metacharacter -- so an unquoted one is de-escaped, split or expanded by the shell,
+# and usermod then names an account that is not the one holding the node shut. The safe set
+# is Python shlex.quote's, so the two halves quote identically.
+_shell_quote() {
+ case "$1" in
+ "") printf "''" ;;
+ *[!A-Za-z0-9_@%+=:,./-]*)
+ printf "'%s'" "$(printf '%s' "$1" | sed "s/'/'\\\\''/g")" ;;
+ *) printf '%s' "$1" ;;
+ esac
+}
+
+# How to open the given nodes, one classified line each, read from the nodes themselves:
+# join:NAME membership WOULD open it -- the group has read and write on the node
+# gid:N no group entry for that GID, the container case docker/run.sh documents,
+# where --group-add passes the host's numeric gids and no name matches.
+# usermod cannot take a bare GID (shadow 4.13: "group '993' does not exist",
+# exit 6), so these are reported rather than prescribed.
+# mode:PATH the mode denies the group too (a udev rule leaving one root:render 0600),
+# so no membership opens it.
+# "render,video" is not universally right and sometimes no group is the answer at all.
+_amd_node_repairs() {
+ for _anr_node in $1; do
+ # acl(5): with an access ACL present the mode's group bits are the ACL MASK, not
+ # the owning group's grant, so the group digit below is an upper bound and
+ # prescribing membership from it is a promise this cannot keep. ls marks such a
+ # node with a trailing "+"; getfacl is not installed everywhere this runs.
+ #
+ # shellcheck disable=SC2012 # not parsing a file LIST: this reads column 11 of the
+ # mode string for one explicitly named node, and the ACL "+" marker is precisely
+ # what find -printf cannot report. The rule is about globbing ls output for names.
+ case "$(ls -ld "$_anr_node" 2>/dev/null | cut -c11)" in
+ +) printf 'acl:%s\n' "$_anr_node"; continue ;;
+ esac
+ # The group NAME goes LAST, because it is the one field NSS controls and the only
+ # one that can carry the separator: winbind hands back DOMAIN\group, and an LDAP
+ # directory can return anything at all. With it anywhere else a name carrying a
+ # pipe shifts every field after it, so the mode, the GID and the uid this branches
+ # on would all be read out of the name. Everything before it comes from the kernel
+ # or from the node list this was called with.
+ stat -c '%a|%g|%u|%n|%G' "$_anr_node" 2>/dev/null || true
+ done | awk -F'|' -v self="$(id -u 2>/dev/null || echo -1)" \
+ -v mygids=" $(id -G 2>/dev/null) " '
+ /^acl:/ { print; next }
+ {
+ # mode|gid|uid|path|group-name. The name is last and is rejoined from every
+ # remaining field, so a name carrying the separator is recovered whole rather
+ # than silently truncated at its first pipe.
+ gname = $5
+ for (i = 6; i <= NF; i++) { gname = gname "|" $i }
+ # A name that cannot be pasted on as ONE group is treated as no name at all,
+ # and the node is reported by its GID instead. usermod -G takes a
+ # comma-separated list, so a group genuinely named "render,sudo" is two groups
+ # to it and the privileged test below, which compares whole names, walks
+ # straight past it. Quoting is the wrong layer: the splitting happens inside
+ # usermod, after the shell has handed it a single argument. The charset is
+ # groupadd(8) portable plus the trailing $ of a Samba machine account, and
+ # mirrors _GROUP_NAME_RE in utils/hardware/amd.py.
+ if (gname !~ /^[A-Za-z_][A-Za-z0-9_.-]*[$]?$/) { gname = "" }
+ # POSIX resolves the owner class EXCLUSIVELY once the uid matches, so on a
+ # node this account owns the group bits are never consulted and no membership
+ # opens it however they read. The repair there is the mode.
+ if (self != -1 && $3 + 0 == self + 0) {
+ # Unless the OWNER digit already grants rw: the node is known shut, so
+ # the mode is not what denies it. Same external denial as the
+ # already-a-member branch, by owner class. Mirrors amd.py external.
+ # Padded first: stat %a drops leading zeros, so mode 060 prints "60" and
+ # an owner index of 0 is not an error -- gawk and mawk return the GROUP
+ # digit, busybox "", so the answer would vary by which awk the host ships.
+ # The group read counts from the right, so it is unaffected.
+ perm = $1
+ while (length(perm) < 3) { perm = "0" perm }
+ u = substr(perm, length(perm) - 2, 1) + 0
+ if (u == 6 || u == 7) { print "external:" $4; next }
+ print "owner:" $4; next
+ }
+ # A record whose GID did not come back as a number is one this cannot reason
+ # about at all, so it is reported as a node no membership opens rather than
+ # branched on. Unreachable while stat answers, and the direction that cannot
+ # invent a repair if it ever stops.
+ if ($2 !~ /^[0-9]+$/) { print "mode:" $4; next }
+ # Neither the owner nor in the owning group leaves this account in the OTHER
+ # class, which POSIX resolves exclusively too: if its digit already grants rw
+ # the mode is not what denies a node already known shut, and no chmod or
+ # usermod moves it. Mirrors the other-class branch in amd.py. Guarded on
+ # membership, since a member is in the group class and the already branch
+ # names the group; above the group-digit test, which would otherwise print a
+ # mode repair for a node the mode is not blocking. The other digit is the
+ # LAST, so unlike the owner one it needs no padding to be read.
+ o = substr($1, length($1), 1) + 0
+ if ((o == 6 || o == 7) && mygids !~ (" " $2 " ")) { print "external:" $4; next }
+ # Group digit of the octal mode; read AND write, since HIP and the Vulkan
+ # loader both open the node read-write.
+ g = substr($1, length($1) - 1, 1) + 0
+ if (g != 6 && g != 7) { print "mode:" $4; next }
+ # Joining one of these hands over a great deal besides the GPU, so a node
+ # owned by one is a udev misconfiguration to report, not a membership to
+ # prescribe. gid 0 as well as the name, since a renamed root group is still
+ # root. Mirrors _PRIVILEGED_GROUPS in utils/hardware/amd.py. ABOVE the unnamed
+ # test: stat prints UNKNOWN for a gid the group database cannot name, so a
+ # minimal container with no entry for gid 0 filed the node as an ordinary
+ # unnamed GID and prescribed groupadd -g 0 plus usermod into root.
+ if ($2 + 0 == 0 || gname ~ /^(root|wheel|sudo|admin|adm|disk|kmem|shadow|docker|lxd)$/) {
+ pname = gname
+ if (pname == "" || pname ~ /^UNKNOWN/) { pname = "root" }
+ if (!pseen[pname]++) print "privileged:" pname
+ next
+ }
+ # The node is already known shut, so a group this account is ALREADY in is
+ # not what denies it: a container device cgroup or an LSM is, and usermod
+ # would exit 0 and change nothing. Read from `id -G`, space-padded so 100
+ # cannot match 1001. Above the unnamed branch for the same reason.
+ if (mygids ~ (" " $2 " ")) {
+ held = gname
+ if (held == "" || held ~ /^UNKNOWN/) { held = $2 }
+ if (!aseen[held]++) print "already:" held
+ next
+ }
+ if (gname == "" || gname ~ /^UNKNOWN/) { if (!gseen[$2]++) print "gid:" $2; next }
+ if (!nseen[gname]++) print "join:" gname
+ }'
+}
+
# rocminfo names each agent twice, so "gfx1201\ngfx1201" is one device, not two.
_amd_probe_arches() {
printf '%s\n' "$1" | sed 's/:.*$//' | tr '[:upper:]' '[:lower:]' | awk 'NF' | sort -u
@@ -4998,6 +5303,17 @@ TORCH_INDEX_URL=$(get_torch_index_url)
_amd_no_rocm_version_reroute=false
_amd_probed_gfx_first=""
+# Read before the branches below and reported after them: a closed node is invisible
+# to every probe this installer runs, so the index gets chosen as if the account could
+# use the card. On a runtime-less host the reroute above then rewrites a */cpu index to
+# a per-arch */gfx* one -- which is exactly the #10466 host -- so answering inside one
+# arm would miss the case this was written for.
+# `|| true` because this whole block is a DIAGNOSTIC and the script runs under `set -e`:
+# an unguarded assignment takes the installer down with it if the helper's last command
+# fails, and the helpers here shell out to stat, awk and tr. On a host missing one of
+# those the answer is "no advice", never "no install". Empty is what every consumer below
+# already treats as nothing to report.
+_closed_amd_nodes="$(_amd_nodes_closed_to_this_user || true)"
case "$TORCH_INDEX_URL" in
*/cpu)
if [ "$_torch_index_pinned" = false ] && [ "$SKIP_TORCH" = false ] && \
@@ -5657,12 +5973,6 @@ case "$TORCH_INDEX_URL" in
substep " driver is current; or run unsloth/scripts/install_rocm_wsl_strixhalo.sh yourself."
else
substep "AMD ROCm users: see https://docs.unsloth.ai/get-started/install-and-update/amd"
- # Only when ROCm truly cannot see the GPU: a detected-but-too-old ROCm (rocminfo works, wheels need 6.0+) has its own guidance.
- if ! _has_amd_rocm_gpu && _amd_gpu_present_via_pci; then
- substep "An AMD GPU is on the PCI bus but ROCm cannot see it (no /dev/kfd," "$C_WARN"
- substep " rocminfo, or amd-smi). Install the ROCm kernel stack so /dev/kfd exists;"
- substep " Strix Halo (gfx1151/gfx1150) needs a recent kernel (6.11+) and ROCm 7.x."
- fi
fi
substep "Re-run with --no-torch for GGUF-only (faster, no PyTorch):"
substep " curl -fsSL https://unsloth.ai/install.sh | sh -s -- --no-torch"
@@ -5676,6 +5986,389 @@ case "$TORCH_INDEX_URL" in
fi
;;
esac
+# These sit after the whole case because they are properties of the HOST, not of the arm
+# the index landed in, and both are gated on the route: _has_amd_rocm_gpu returns false on
+# ANY host with a usable NVIDIA GPU, so without the gate every hybrid box with an AMD card
+# on the bus would be told to install a ROCm kernel stack its CUDA wheels never use.
+#
+# Two sibling arms rather than a nested `if`, since an ABSENT /dev/kfd is not a CLOSED one
+# and a host can need both repairs; the harnesses also lift this block by walking back to
+# the `if` above a sentence.
+#
+# The runtime half's needs_kfd: SKIP_TORCH does not settle it, because --no-torch still
+# installs a GGUF bundle and the ROCm one opens /dev/kfd exactly as torch would (the #10466
+# shape). The three named are the REQUESTABLE_BACKENDS that are not ROCm
+# (utils/prebuilt/llama_backend.py). Normalized as the bundle selector normalizes it
+# (studio/setup.sh, `awk '{$1=$1}'`): deleting internal whitespace let "vul kan" match here
+# and go quiet while setup.sh rejects it and may fall back to ROCm.
+
+# Whether the TORCH this run installs can open an AMD device node, which only a ROCm wheel
+# does. Asked positively, not as "anything that is not the cpu leaf": that read a CUDA
+# index as a KFD consumer, and told a healthy hybrid host to repair /dev/kfd permissions
+# for a node neither its CUDA wheel nor its Vulkan bundle opens. Classified exactly as the
+# route case classifies, so the two cannot disagree.
+_torch_opens_amd_nodes() {
+ [ "$SKIP_TORCH" = true ] && return 1
+ _toan_leaf=$(_torch_index_url_leaf "${TORCH_INDEX_URL:-}")
+ case "$_toan_leaf" in
+ # repo.radeon.com is rocm-rel-X.Y and nothing else; a pin that merely starts with
+ # it is somebody's mirror, exactly as the route case anchors its own arm.
+ rocm-rel-*[!0-9.]*) return 1 ;;
+ rocm-rel-[0-9]*) return 0 ;;
+ esac
+ _is_pip_rocm_family_leaf "$_toan_leaf" && return 0
+ return 1
+}
+
+# An unset or `auto` request is not a decision here, but it is not a coin toss either:
+# _linux_published_attempts takes the CUDA bundle under `if host.has_usable_nvidia:` and
+# only reaches ROCm in the `elif host.has_rocm` below it. So on a hybrid box the automatic
+# bundle opens no AMD node, and the AMD-evidence gates cannot tell: the card IS there and
+# its nodes ARE shut, they are simply nothing this install will use. Listed as the values
+# that ARE decisions rather than the two that are not, because setup.sh normalises anything
+# outside this set away (is_requestable_backend -> None -> auto).
+_auto_bundle_opens_amd_nodes() {
+ case "$(_requested_llama_backend)" in
+ cpu|cuda|rocm|hip|vulkan) return 0 ;;
+ esac
+ # Memoized: _has_usable_nvidia_gpu shells out to `nvidia-smi -L` and the two scope
+ # predicates below ask repeatedly, so an NVIDIA-less host paid one probe per gate.
+ # Nothing it reads changes within a run.
+ if [ -z "${_amd_auto_nvidia_cached:-}" ]; then
+ if _has_usable_nvidia_gpu; then
+ _amd_auto_nvidia_cached=yes
+ else
+ _amd_auto_nvidia_cached=no
+ fi
+ fi
+ [ "$_amd_auto_nvidia_cached" = yes ] && return 1
+ return 0
+}
+
+# The effective llama.cpp backend request, resolved the way
+# utils/prebuilt/llama_backend.py::environment_backend_override resolves it: a RECOGNISED
+# UNSLOTH_LLAMA_CPP_BACKEND wins outright ("auto" included, being a request to detect), and
+# only an absent or unrecognised one leaves the legacy UNSLOTH_FORCE_VULKAN in effect.
+# Reading the new variable alone put UNSLOTH_FORCE_VULKAN=1 on the automatic route here while
+# effective_backend_request built a Vulkan bundle, so on a CUDA-indexed or hybrid host the
+# render-node diagnoses were suppressed for a run that opens exactly those nodes.
+_requested_llama_backend() {
+ _rlb=$(printf '%s' "${UNSLOTH_LLAMA_CPP_BACKEND:-}" | awk '{$1=$1; print tolower($0)}')
+ case "$_rlb" in
+ cpu|cuda|rocm|hip|vulkan|auto) printf '%s\n' "$_rlb"; return 0 ;;
+ esac
+ case "$(printf '%s' "${UNSLOTH_FORCE_VULKAN:-}" | awk '{$1=$1; print tolower($0)}')" in
+ 1|true|yes|on) printf '%s\n' vulkan; return 0 ;;
+ esac
+ # An unrecognised value falls through unchanged, so the callers' cases miss it and the
+ # automatic route decides, exactly as setup.sh normalises a typo away to auto.
+ printf '%s\n' "$_rlb"
+}
+
+_run_may_open_kfd() {
+ _torch_opens_amd_nodes && return 0
+ case "$(_requested_llama_backend)" in
+ vulkan|cpu|cuda) return 1 ;;
+ esac
+ _auto_bundle_opens_amd_nodes || return 1
+ return 0
+}
+
+# One layer wider, for the diagnoses that are not about /dev/kfd. A Vulkan bundle opens a
+# render node, so those still apply to it; a CPU or CUDA bundle beside --no-torch opens no
+# AMD node at all, and telling that install to join the render group describes a card
+# nothing in the run was going to touch.
+_run_may_open_a_gpu_node() {
+ _torch_opens_amd_nodes && return 0
+ case "$(_requested_llama_backend)" in
+ cpu|cuda) return 1 ;;
+ esac
+ _auto_bundle_opens_amd_nodes || return 1
+ return 0
+}
+
+# Read from the LEAF, and through the same _is_pip_rocm_family_leaf every other index
+# classifier here uses: a whole-URL */rocm*|*/gfx* glob also matches a custom pin whose
+# final segment merely STARTS with one ("gfx-mirror", "rocm7.2-private"), which that helper
+# exists to reject, and a hybrid host on such a pin then got repairs for a card its wheels
+# have nothing to do with. Recomputed from TORCH_INDEX_URL rather than reusing
+# $_torch_index_leaf, since the per-arch reroutes rewrite the URL after that is set.
+# repo.radeon.com is named separately: its leaf is rocm-rel-X.Y, a real ROCm route that is
+# not a pip family, so the family test alone would drop it.
+# Guarded for the same reason as the closed-node read above: a diagnostic may not
+# abort the install under `set -e`. An empty leaf falls to the pip-family arm below.
+_amd_node_diag_leaf=$(_torch_index_url_leaf "$TORCH_INDEX_URL" || true)
+case "$_amd_node_diag_leaf" in
+ # A repo.radeon.com leaf is rocm-rel-X.Y[.Z] and nothing else, anchored the way
+ # _is_pip_rocm_family_leaf anchors its own rocm[0-9]* arm: a pin that merely STARTS
+ # with it (rocm-rel-7.0-private) is somebody's mirror. The gfx family is deliberately
+ # NOT narrowed this way, since AMD's own indexes are gfx110X-all, gfx120X-all,
+ # gfx103X-all, where a suffix is the convention rather than a custom pin.
+ rocm-rel-*[!0-9.]*) _amd_node_diag_route=false ;;
+ cpu|rocm-rel-[0-9]*) _amd_node_diag_route=true ;;
+ *)
+ if _is_pip_rocm_family_leaf "$_amd_node_diag_leaf"; then
+ _amd_node_diag_route=true
+ else
+ _amd_node_diag_route=false
+ fi
+ ;;
+esac
+# ... but only when a wheel is actually being installed. TORCH_INDEX_URL is resolved
+# unconditionally above, so --no-torch on a hybrid or CUDA-pinned host read as a CUDA route
+# and silenced all three diagnoses for a run whose bundle opens the very nodes they are
+# about. `auto` is deliberately included: the AMD-evidence gates below are what keep an
+# NVIDIA-only host silent, so routing here fails closed rather than guessing a backend.
+if [ "$SKIP_TORCH" = true ]; then
+ if _run_may_open_a_gpu_node; then
+ _amd_node_diag_route=true
+ else
+ _amd_node_diag_route=false
+ fi
+fi
+# ... and an EXPLICIT GPU bundle request opens AMD nodes whatever the wheels do. The route
+# above is derived from the torch index alone, so a CUDA or custom-pinned index asked for
+# the rocm or vulkan bundle read as false, and the SKIP_TORCH override could not catch it
+# because that only runs when no wheel is installed. This only ever turns the route ON: a
+# cpu or cuda request is still settled by the two scope predicates below.
+case "$(_requested_llama_backend)" in
+ rocm|hip|vulkan) _amd_node_diag_route=true ;;
+esac
+# Separate branches, not one branch with an inner test, because they need DIFFERENT
+# evidence. The mapping one is gated on the KFD topology, the amdkfd driver's own sysfs, so
+# it must not sit behind _has_amd_rocm_gpu: that probe answers from `amd-smi list`, which
+# reads the driver over sysfs and libdrm and so SUCCEEDS in a container given only
+# --device /dev/dri, where HIP has no /dev/kfd to open (llama_cpp.py's
+# _rocm_hip_is_reachable records the same disagreement). Behind that probe the warning was
+# suppressed on exactly the container shape it was written for.
+if [ "$_amd_node_diag_route" = true ] && \
+ _run_may_open_kfd && [ "$OS" != "macos" ] && \
+ [ ! -e /dev/kfd ] && _amd_silicon_behind_a_missing_kfd; then
+ substep "An AMD GPU is in the KFD topology but /dev/kfd is not present, so the" "$C_WARN"
+ substep " driver is loaded and reinstalling ROCm changes nothing: the node itself"
+ substep " is missing. Under Docker, recreate the container with --device /dev/kfd"
+ substep " --device /dev/dri; on a bare host it is a udev or devtmpfs problem."
+elif [ "$_amd_node_diag_route" = true ] && \
+ _run_may_open_kfd && [ "$OS" != "macos" ] && \
+ ! printf '%s\n' "$_closed_amd_nodes" | grep -qx /dev/kfd && \
+ ! _has_amd_rocm_gpu ignore-nvidia && _amd_gpu_present_via_pci && \
+ [ -e /dev/kfd ] && _kfd_node_is_amds; then
+ substep "An AMD GPU is on the PCI bus and /dev/kfd is present and openable, so" "$C_WARN"
+ substep " the kernel stack is already loaded and reinstalling it changes nothing."
+ substep " What is missing is the ROCm userspace that reads the card: install"
+ substep " rocminfo and amd-smi (rocminfo, rocm-smi-lib) and re-run. Strix Halo"
+ substep " (gfx1151/gfx1150) also needs a recent kernel (6.11+) and ROCm 7.x."
+elif [ "$_amd_node_diag_route" = true ] && \
+ _run_may_open_kfd && [ "$OS" != "macos" ] && \
+ ! printf '%s\n' "$_closed_amd_nodes" | grep -qx /dev/kfd && \
+ ! _has_amd_rocm_gpu ignore-nvidia && _amd_gpu_present_via_pci && \
+ { [ ! -e /dev/kfd ] || ! _kfd_node_is_amds; }; then
+ substep "An AMD GPU is on the PCI bus but ROCm cannot see it (no /dev/kfd," "$C_WARN"
+ substep " rocminfo, or amd-smi). Install the ROCm kernel stack so /dev/kfd exists;"
+ substep " Strix Halo (gfx1151/gfx1150) needs a recent kernel (6.11+) and ROCm 7.x."
+fi
+if ! _run_may_open_kfd; then
+ _closed_amd_nodes=$(printf '%s\n' "$_closed_amd_nodes" | grep -vx /dev/kfd || true)
+fi
+# The driver is loaded and the nodes exist, so neither a wheel nor a kernel stack
+# repairs this; only group membership does. Nothing else in this installer asks
+# whether the account can OPEN a node it just found (#10466). /dev/kfd alone stops
+# ROCm; a render node stops Vulkan as well, so the two are not claimed together.
+if [ "$_amd_node_diag_route" = true ] && _run_may_open_a_gpu_node && \
+ [ -n "$_closed_amd_nodes" ]; then
+ substep "An AMD GPU is present but this account cannot open its device nodes:" "$C_WARN"
+ printf '%s\n' "$_closed_amd_nodes" | while IFS= read -r _n; do
+ substep " $_n"
+ done
+ # Which backends the closed set blocks. The membership sentence is NOT here: when
+ # every refused node has an unnamed GID, an ACL, or a mode no group can open,
+ # _amd_node_repairs deliberately names no group, and this used to leave "Add yourself
+ # to the" hanging above a branch explaining that no membership opens the node.
+ if printf '%s\n' "$_closed_amd_nodes" | grep -qv '^/dev/kfd$'; then
+ if _an_amd_render_node_is_open; then
+ # The repair still stands -- these nodes are still shut -- but the claim does
+ # not: some AMD render node here already opens, so a backend that enumerates
+ # every device has a path and it is the card behind THESE nodes that is out of
+ # reach. amd_node_permission_hint draws the same distinction on the Python side.
+ substep " Every backend needs them, ROCm and Vulkan alike, but another AMD"
+ substep " render node on this host is open, so what they block is the card"
+ substep " behind them rather than all GPU work."
+ else
+ substep " Every backend needs them, ROCm and Vulkan alike."
+ fi
+ else
+ substep " ROCm needs it; Vulkan does not."
+ fi
+ # Read from the nodes that were refused, so the advice matches those files.
+ # Guarded: `set -e` plus an unguarded substitution would let a missing awk abort
+ # the installer from inside the advice it was about to print.
+ _closed_amd_repairs=$(_amd_node_repairs "$_closed_amd_nodes" || true)
+ _closed_amd_groups=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^join://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ _closed_amd_gids=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^gid://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ _closed_amd_modes=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^mode://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ _closed_amd_acls=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^acl://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ _closed_amd_owned=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^owner://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ _closed_amd_priv=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^privileged://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ _closed_amd_already=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^already://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ _closed_amd_external=$(printf '%s\n' "$_closed_amd_repairs" | sed -n 's/^external://p' \
+ | tr '\n' ',' | sed 's/,*$//')
+ # Who the mode tests above answered for. $USER is inherited, so a container that changes
+ # its numeric user without resetting it names somebody else, and the usermod below would
+ # then modify an account that is not the one holding the device shut.
+ #
+ # `id -un` FAILS for a uid with no passwd entry, which is the ordinary shape of
+ # `docker run --user 1234`, and the inherited USER there commonly still says root. So
+ # empty means "no account to name" and the branches below print the container repair
+ # instead of a usermod that would succeed against an identity nothing is running as.
+ # Status, not stdout: GNU id PRINTS the uid and then exits 1 for an unresolvable one
+ # (coreutils id.c print_user falls back to uidtostr), so `|| printf ''` never runs and
+ # the substitution captured "1234" -- the numeric form usermod rejects, prescribed on
+ # exactly the container hosts the paragraph above is about.
+ _amd_repair_user=$(id -un 2>/dev/null) || _amd_repair_user=''
+ # The documented pair is the fallback for nodes that could not be stat'd at all, where
+ # some advice beats none. A node that WAS read and offers no joinable group gets the
+ # sentences below instead of a command that would fail.
+ if [ -z "$_closed_amd_groups" ] && [ -z "$_closed_amd_gids" ] && \
+ [ -z "$_closed_amd_modes" ] && [ -z "$_closed_amd_acls" ] && \
+ [ -z "$_closed_amd_owned" ] && [ -z "$_closed_amd_priv" ] && \
+ [ -z "$_closed_amd_already" ] && [ -z "$_closed_amd_external" ]; then
+ _closed_amd_groups="render,video"
+ fi
+ if [ -n "$_closed_amd_groups" ] && [ -z "$_amd_repair_user" ]; then
+ _closed_amd_group_adds=$(printf '%s\n' "$_closed_amd_groups" | tr ',' '\n' \
+ | while IFS= read -r _cga_name; do
+ [ -n "$_cga_name" ] || continue
+ printf -- '--group-add %s ' "$(_shell_quote "$_cga_name")"
+ done | sed 's/ *$//')
+ substep " This uid has no passwd entry, so usermod has no account to name:" "$C_WARN"
+ substep " recreate the container passing $_closed_amd_group_adds, or run it"
+ substep " as an account this system knows."
+ elif [ -n "$_closed_amd_groups" ]; then
+ case "$_closed_amd_groups" in
+ *,*) substep " Add yourself to the $_closed_amd_groups groups, then log out" ;;
+ *) substep " Add yourself to the $_closed_amd_groups group, then log out" ;;
+ esac
+ substep " and back in:"
+ substep " sudo usermod -a -G $(_shell_quote "$_closed_amd_groups") $(_shell_quote "$_amd_repair_user")"
+ fi
+ if [ -n "$_closed_amd_gids" ]; then
+ # One flag per GID, as docker/run.sh does and as the Python half already emits:
+ # --group-add takes a SINGLE value, so a comma-joined pair is one group name that
+ # does not exist, and naming only the first leaves the second node shut.
+ _closed_amd_gid_adds=$(printf '%s' "$_closed_amd_gids" | tr ',' '\n' \
+ | sed 's/^/--group-add /' | tr '\n' ' ' | sed 's/ *$//')
+ # groupadd alone only gives the numeric owner a NAME: the account is still
+ # outside the group and the node is still shut, so both halves are printed -- and
+ # one pair PER GID, since one groupadd names one owner and the sentence above
+ # already says "create a group for each".
+ case "$_closed_amd_gids" in
+ *,*) substep " Some of those nodes belong to GIDs $_closed_amd_gids, which have no" "$C_WARN"
+ substep " group entry here, so usermod cannot name them: create a group for each" ;;
+ *) substep " Some of those nodes belong to GID $_closed_amd_gids, which has no" "$C_WARN"
+ substep " group entry here, so usermod cannot name it: create a group for it" ;;
+ esac
+ if [ -n "$_amd_repair_user" ]; then
+ substep " and add yourself to every one of them, then log out and back in:"
+ for _amd_gid in $(printf '%s' "$_closed_amd_gids" | tr ',' ' '); do
+ # Generated, not a <name> placeholder: this is a command to paste, and angle
+ # brackets are redirection operators, so `groupadd -g 993 <name>` is a syntax
+ # error before groupadd runs. Keyed on the GID, which has no entry by
+ # definition here -- which says nothing about the NAME, and some host may
+ # already have taken it at a different GID.
+ _amd_gid_name="amdgpu$_amd_gid"
+ # Chained, so that collision cannot become a silent wrong repair: printed as
+ # two separate lines, a failed groupadd is followed by a usermod that
+ # SUCCEEDS against the wrong group and leaves the node exactly as shut,
+ # having reported success. && stops there, and the error names the cause.
+ substep " sudo groupadd -g $_amd_gid $_amd_gid_name && \\"
+ substep " sudo usermod -a -G $_amd_gid_name $(_shell_quote "$_amd_repair_user")"
+ done
+ substep " or recreate the container passing $_closed_amd_gid_adds."
+ else
+ # The bare-host half needs an account to add and there is none, so the container
+ # half is the whole repair for this shape.
+ substep " and this uid has no passwd entry either, so neither groupadd nor"
+ substep " usermod has anything to name: recreate the container passing"
+ substep " $_closed_amd_gid_adds."
+ fi
+ fi
+ if [ -n "$_closed_amd_modes" ]; then
+ substep " $_closed_amd_modes does not grant its own group read and write, so no" "$C_WARN"
+ substep " membership opens it: fix the udev rule or the node's permissions."
+ fi
+ if [ -n "$_closed_amd_already" ]; then
+ case "$_closed_amd_already" in
+ *,*) substep " This account is already in the $_closed_amd_already groups that own" "$C_WARN" ;;
+ *) substep " This account is already in the $_closed_amd_already group that owns" "$C_WARN" ;;
+ esac
+ substep " those nodes, so usermod would change nothing: something outside the"
+ substep " file mode is denying them, typically a container device cgroup or an"
+ substep " LSM such as SELinux or AppArmor."
+ fi
+ if [ -n "$_closed_amd_owned" ]; then
+ substep " $_closed_amd_owned is owned by this account, and POSIX stops at the" "$C_WARN"
+ substep " owner bits once the uid matches, so no group membership opens it"
+ substep " however its group bits read: fix the mode, or the udev rule behind it."
+ fi
+ if [ -n "$_closed_amd_external" ]; then
+ # Worded by the class that APPLIES, not by the owner one: this bucket holds a node
+ # this account owns whose owner bits grant rw AND one it neither owns nor shares a
+ # group with whose other bits do, and naming the owner for both told the second kind
+ # it owns a node it does not. Mirrors the external sentence in amd.py.
+ substep " $_closed_amd_external is already granted read and write by the" "$C_WARN"
+ substep " permission bits that apply to this account, so the mode is not what"
+ substep " is shutting it:"
+ substep " something outside the file mode is denying it, typically a container"
+ substep " device cgroup or an LSM such as SELinux or AppArmor."
+ fi
+ if [ -n "$_closed_amd_priv" ]; then
+ substep " Those nodes belong to the $_closed_amd_priv group, which grants a" "$C_WARN"
+ substep " great deal besides the GPU, so joining it is not the repair: fix the"
+ substep " udev rule so the node is owned by render or video instead."
+ fi
+ if [ -n "$_closed_amd_acls" ]; then
+ substep " $_closed_amd_acls carries a POSIX ACL, so the group permissions cannot" "$C_WARN"
+ substep " be read from its mode: check the real grant with getfacl before"
+ substep " changing group membership."
+ fi
+ # The container shape of the missing-node problem: /dev/kfd mapped without
+ # /dev/dri leaves the closed KFD node looking like the whole story while ROCr
+ # has no render node to open, and no group creates one.
+ if ! _amd_render_node_present; then
+ # The nodes THIS run opens: a Vulkan bundle beside --no-torch never opens /dev/kfd,
+ # so naming it hands the container another host device for nothing.
+ if _run_may_open_kfd; then
+ _amd_map_devices="--device /dev/kfd --device /dev/dri"
+ else
+ _amd_map_devices="--device /dev/dri"
+ fi
+ substep " No AMD render node (/dev/dri/renderD*) is present either, and ROCm and" "$C_WARN"
+ substep " Vulkan both open one, so the device mapping needs fixing too; under"
+ substep " Docker that is $_amd_map_devices."
+ fi
+# The same missing render node with nothing closed, which is the ordinary container shape
+# of it: --device /dev/kfd and no --device /dev/dri leaves one openable node, so the list
+# above is empty and this went unsaid. Gated on the KFD topology naming AMD rather than on
+# the glob being empty, since every vendor's render nodes live under it.
+elif [ "$_amd_node_diag_route" = true ] && _run_may_open_a_gpu_node && \
+ [ "$OS" != "macos" ] && \
+ ! _amd_render_node_present && _kfd_topology_has_an_amd_gpu; then
+ if _run_may_open_kfd; then
+ _amd_map_devices="--device /dev/kfd --device /dev/dri"
+ else
+ _amd_map_devices="--device /dev/dri"
+ fi
+ substep "An AMD GPU is in the KFD topology but no AMD render node" "$C_WARN"
+ substep " (/dev/dri/renderD*) is present, and ROCm and Vulkan both open one, so the"
+ substep " device mapping needs fixing; under Docker that is $_amd_map_devices."
+fi
# ── Install unsloth directly into the venv (no activation needed) ──
tauri_log "STEP" "Installing PyTorch"
diff --git a/studio/backend/core/inference/llama_cpp.py b/studio/backend/core/inference/llama_cpp.py
index 592fdf37805..e7f8a4ff295 100644
--- a/studio/backend/core/inference/llama_cpp.py
+++ b/studio/backend/core/inference/llama_cpp.py
@@ -12599,10 +12599,184 @@ class LlamaCppBackend:
if _metal_capable_host():
# Same check as the load site: an Intel Mac wants its real reason.
return "this probe reads CUDA and HIP only; Apple Silicon offloads through Metal"
- if LlamaCppBackend._is_vulkan_backend(binary):
- return "the Vulkan probe reported no device"
+ # Before the backend branches, because it is the reason underneath BOTH: a
+ # render node this user cannot open leaves HIP with no device and the Vulkan
+ # loader with nothing to enumerate, and "the Vulkan probe reported no device"
+ # then sends the user after a driver that is fine (#10466). A Vulkan binary is
+ # asked only about the render node, never /dev/kfd, and keeps its own reason.
+ # Only of a build that can drive an AMD card: _is_vulkan_backend answers which
+ # backend the install defers to, so a CUDA-plus-Vulkan build counts as CUDA. An
+ # install this probe cannot read stays eligible, so a detection miss does not
+ # lose the #10466 host.
+ _is_vulkan = LlamaCppBackend._is_vulkan_backend(binary)
+ _backends = LlamaCppBackend._installed_ggml_backends(binary)
+ _amd_capable = not LlamaCppBackend._backend_lacks_gpu_lib(binary) and (
+ _is_vulkan or "hip" in _backends or not _backends
+ )
+
+ # The ordinal space the three device lists index, so an entry can be checked
+ # against something rather than only read. None on any host whose KFD topology
+ # cannot be read, which is the direction that leaves a selector alone.
+ try:
+ from utils.hardware.amd import amd_kfd_gpu_node_count
+ _amd_gpu_count = amd_kfd_gpu_node_count()
+ except Exception: # noqa: BLE001
+ _amd_gpu_count = None
+
+ def _post_rocr_device_count() -> "int | None":
+ # ROCr filters the physical list FIRST and renumbers what survives, and
+ # the HIP layer indexes those (_rocm_visibility_masks_are_stacked), so a
+ # HIP ordinal is judged against the post-ROCr count: on two GPUs with
+ # ROCR_VISIBLE_DEVICES=0 one survives, and HIP ordinal 1 hides everything
+ # where the physical count of 2 reads it as harmless. None when the
+ # survivors cannot be counted, which leaves the HIP selector alone.
+ if not _rocr_filters:
+ return _amd_gpu_count
+ _raw = (os.environ.get("ROCR_VISIBLE_DEVICES") or "").strip()
+ if not _raw or not _amd_gpu_count:
+ return None
+ # ROCr's own rule, from RvdFilter in core/inc/amd_filter_device.h: it
+ # surfaces "tokens that are Legal and NOT Terminating", an index terminates
+ # when it "lies outside the interval [0 - (numGpuDevices - 1)]" OR "maps to
+ # a device that has been previously selected", and a token is Illegal when
+ # it "can't be evaluated into an instance of Device UUID or Enumeration
+ # Index". Every ending is a PREFIX of known length, including a repeated
+ # ordinal ("0,0" surfaces one device) and an empty token. Only a UUID is
+ # unknowable here: the KFD count is an ordinal space.
+ _survivors = 0
+ _selected: "set[int]" = set()
+ for _entry in _raw.split(","):
+ _entry = _entry.strip()
+ if not _entry.isdigit():
+ # AMD documents the UUID form as the literal "GPU-XX"; anything
+ # else that is not an index is Illegal to ROCr too, so it ends the
+ # list at a length this does know.
+ return None if _entry.lower().startswith("gpu-") else _survivors
+ _idx = int(_entry)
+ if _idx >= _amd_gpu_count or _idx in _selected:
+ return _survivors
+ _selected.add(_idx)
+ _survivors += 1
+ return _survivors
+
+ def _hides_every_device(
+ value: str,
+ count: "int | None" = None,
+ *,
+ strict: bool = False,
+ ) -> bool:
+ # CUDA and HIP read the list left to right and stop at the first entry
+ # that names no device, so a value that is empty, or whose FIRST entry
+ # is empty or negative, exposes nothing; HIP_VISIBLE_DEVICES=0 still
+ # exposes GPU 0.
+ first = value.split(",")[0].strip()
+ if first == "" or first.startswith("-"):
+ return True
+ # Nor does a token have to LOOK like a number to end the list. clr takes
+ # `index = atoi(str_id)` and rejects the token unless `str_id` is that
+ # index written back out, so HIP_VISIBLE_DEVICES=garbage (and 0x1, and 00)
+ # terminates on the FIRST token, exactly as -1 does. Asked only for the
+ # clr-layer variables: ROCr's illegal-token rule is a separate parser, in
+ # _is_an_illegal_rocr_selector, and a UUID token is left to
+ # _cannot_be_resolved since resolving it needs the agents.
+ if strict and not first.lower().startswith("gpu-"):
+ try:
+ _index = int(first)
+ except ValueError:
+ return True
+ if str(_index) != first:
+ return True
+ # An entry that looks valid can still name nothing: the list stops at the
+ # first index no device answers to, so HIP_VISIBLE_DEVICES=3 on a one-GPU
+ # host exposes zero devices, which is the empty probe being explained. Only
+ # ordinals, and only against a count actually read: reading an unknown
+ # count as a bound would call every selector here a blocker.
+ _bound = _amd_gpu_count if count is None else count
+ if not _bound or not first.isdigit():
+ return False
+ return int(first) >= _bound
+
+ def _cannot_be_resolved(value: str) -> bool:
+ # ROCr and clr both resolve a UUID ("0,GPU-4b2c...") as well as an
+ # ordinal (rocdevice.cpp matches "GPU-" against HSA_AMD_AGENT_INFO_UUID).
+ # Nothing here can match one, so it is unresolved rather than judged:
+ # calling it a blocker invents a fault. Every other non-index is Illegal,
+ # which _is_an_illegal_rocr_selector decides.
+ first = value.split(",")[0].strip()
+ return first.lower().startswith("gpu-")
+
+ def _vk_selects_no_device(value: str) -> bool:
+ # ggml stops at the first token with no integer prefix, so if the FIRST
+ # lacks one device_indices stays empty. A sign counts as a prefix: "-1"
+ # extracts and wraps, which throws rather than selecting nothing.
+ _first = value.replace(",", " ").split()
+ if not _first:
+ return True
+ _token = _first[0]
+ _digits = _token[1:] if _token[:1] in ("+", "-") else _token
+ return not _digits[:1].isdigit()
+
+ def _vk_ordinal_always_throws(value: str) -> bool:
+ # A negative ordinal is the one out-of-range value decidable WITHOUT the
+ # raw device count, because it wraps past every possible one: ggml reads
+ # `size_t tmp; while (ss >> tmp)` (ggml-vulkan.cpp, ggml_vk_instance_init),
+ # strtoull accepts the sign, and "-1" becomes 2**64-1, so it always throws
+ # "Invalid Vulkan device index". No group membership repairs that, so it is
+ # a blocker like an Illegal ROCr token. Extraction walks the list, so a
+ # negative one throws wherever it sits provided every token before it still
+ # extracts; "-0" wraps to 0 and is in range.
+ for _token in value.replace(",", " ").split():
+ _signed = _token[:1] in ("+", "-")
+ _digits = _token[1:] if _signed else _token
+ if not _digits[:1].isdigit():
+ return False # extraction stops here, so nothing after it is read
+ if _token[:1] == "-" and _digits.lstrip("0"):
+ return True
+ return False
+
+ def _is_an_illegal_rocr_selector(value: str) -> bool:
+ # ROCr's filter (ROCR-Runtime, core/inc/amd_filter_device.h) calls a token
+ # Illegal when it "can't be evaluated into an instance of Device UUID or
+ # Enumeration Index", and an Illegal token terminates the list -- so an
+ # illegal FIRST token leaves zero survivors, exactly as an out-of-range
+ # ordinal does, and _post_rocr_device_count already counts it that way.
+ # Reported as a definite blocker rather than as something to check only if
+ # the group change fails, since no membership makes the runtime enumerate
+ # a device again.
+ first = value.split(",")[0].strip()
+ return (
+ bool(first)
+ and not first.startswith("-")
+ and not first.isdigit()
+ and not first.lower().startswith("gpu-")
+ )
+
+ # Which of the four this host actually reads, per variable rather than one rule
+ # for all. The runtime being explained is HIP, so clr's precedence holds:
+ # rocdevice.cpp reads HIP_VISIBLE_DEVICES when its FIRST BYTE is not NUL and
+ # CUDA_VISIBLE_DEVICES otherwise, so an empty CUDA mask behind a valid HIP one
+ # is never consulted -- while an empty HIP mask does not win, clr's flag
+ # defaulting to "" and unable to tell it from unset. ROCr sits BELOW that layer
+ # and composes rather than defers (_rocm_visibility_masks_are_stacked), so an
+ # empty ROCr mask does blind the runtime; Windows has no ROCr layer at all.
+ # _active_gpu_visibility_mask is deliberately not used: it gates the same chain
+ # on torch being a ROCm build, which is the wrong question for a HIP
+ # llama-server sitting beside the CPU torch wheel this host tends to have.
+ _hip_layer_var = (
+ "HIP_VISIBLE_DEVICES"
+ if os.environ.get("HIP_VISIBLE_DEVICES", "")
+ else "CUDA_VISIBLE_DEVICES"
+ )
+ _rocr_filters = (
+ sys.platform != "win32" and os.environ.get("ROCR_VISIBLE_DEVICES") is not None
+ )
+ _ordinal_filters = LlamaCppBackend._gpu_device_ordinal_active()
+
+ _post_rocr_count = _post_rocr_device_count()
masks = []
+ blocking = []
+ unresolved = []
for var in (
"CUDA_VISIBLE_DEVICES",
"HIP_VISIBLE_DEVICES",
@@ -12610,35 +12784,201 @@ class LlamaCppBackend:
"GPU_DEVICE_ORDINAL",
):
raw = os.environ.get(var)
- if raw is not None:
- masks.append(f"{var}={raw!r}" if raw.strip() else f"{var} is empty")
+ if raw is None:
+ continue
+ phrase = f"{var}={raw!r}" if raw.strip() else f"{var} is empty"
+ masks.append(phrase)
+ if var == "GPU_DEVICE_ORDINAL":
+ _consulted = _ordinal_filters
+ elif var == "ROCR_VISIBLE_DEVICES":
+ _consulted = _rocr_filters
+ else:
+ _consulted = var == _hip_layer_var
+ # A Vulkan build reads none of the four, so none of them blocks it.
+ if _is_vulkan or not _consulted:
+ continue
+ # ROCr indexes the physical list, the HIP layer indexes ROCr's survivors.
+ _rocr = var == "ROCR_VISIBLE_DEVICES"
+ _bound = _amd_gpu_count if _rocr else _post_rocr_count
+ if _hides_every_device(raw, _bound, strict = not _rocr) or (
+ _rocr and _is_an_illegal_rocr_selector(raw)
+ ):
+ blocking.append(phrase)
+ elif _cannot_be_resolved(raw):
+ unresolved.append(phrase)
+ # The four above are the HIP/CUDA selectors, which a Vulkan build reads none of.
+ # GGML_VK_VISIBLE_DEVICES is the one it DOES read and _run_vulkan_probe passes
+ # it through deliberately, so it is the only selector that can empty a Vulkan
+ # probe. Only the two ends are decidable here: ggml_vk_instance_init reads
+ # ordinals with `ss >> tmp` against the RAW vkEnumeratePhysicalDevices list,
+ # before CPU devices are dropped and ICDs deduplicated, so this process does not
+ # have the bound. A first token with no integer prefix (the empty string
+ # included) selects nothing, and a NEGATIVE ordinal wraps past every possible
+ # bound and always throws; a merely large positive one needs the bound to judge,
+ # so it stays unresolved. Anything else is reported as unresolved.
+ if _is_vulkan:
+ _vk_raw = os.environ.get("GGML_VK_VISIBLE_DEVICES")
+ if _vk_raw is not None:
+ _vk_phrase = (
+ f"GGML_VK_VISIBLE_DEVICES={_vk_raw!r}"
+ if _vk_raw.strip()
+ else "GGML_VK_VISIBLE_DEVICES is empty"
+ )
+ masks.append(_vk_phrase)
+ if _vk_selects_no_device(_vk_raw) or _vk_ordinal_always_throws(_vk_raw):
+ blocking.append(_vk_phrase)
+ else:
+ unresolved.append(_vk_phrase)
+
mask_note = f" ({', '.join(masks)})" if masks else ""
+ node_hint = None
+ if _amd_capable:
+ try:
+ from utils.hardware.amd import amd_node_permission_hint
+ node_hint = amd_node_permission_hint(needs_kfd = not _is_vulkan)
+ except Exception: # noqa: BLE001
+ node_hint = None
+
+ def _closed_nodes_block_the_runtime() -> bool:
+ # A host this cannot read answers True, which keeps the closed node as the
+ # stated reason -- what this returned before the sibling check existed.
+ try:
+ from utils.hardware.amd import amd_closed_nodes_block_the_runtime
+ return amd_closed_nodes_block_the_runtime(needs_kfd = not _is_vulkan)
+ except Exception: # noqa: BLE001
+ return True
+
+ def _another_vendor_has_an_open_node() -> bool:
+ # Vulkan enumerates any vendor, so an open Intel or NVIDIA render node is a
+ # complete path for THIS binary and the closed AMD one cannot be the whole
+ # story. Asked only for Vulkan: HIP needs /dev/kfd and an AMD render node,
+ # which no other vendor's node substitutes for. False on a host this cannot
+ # read, which keeps the behaviour it had before the check existed.
+ if not _is_vulkan:
+ return False
+ # Which drivers the loader would actually load decides this: a loader that
+ # can only load AMD never opens the other vendor's driver, so its open node
+ # is no path. One that can load that vendor is the opposite case and must
+ # NOT suppress.
+ try:
+ from utils.hardware.amd import (
+ a_non_amd_render_node_is_open,
+ the_vulkan_loader_can_only_load_amd,
+ )
+ if the_vulkan_loader_can_only_load_amd():
+ return False
+ return a_non_amd_render_node_is_open()
+ except Exception: # noqa: BLE001
+ return False
+
+ # The closed node when it does NOT explain the empty probe, appended to whatever
+ # reason does rather than returned in place of it.
+ _second_finding = ""
+ # Sufficient on its own, so it rides on the PRIMARY reason, not on what
+ # gets demoted.
+ _loader_finding = ""
+
+ def _the_vulkan_loader_has_no_driver() -> bool:
+ try:
+ from utils.hardware.amd import the_vulkan_loader_has_no_usable_driver
+ return the_vulkan_loader_has_no_usable_driver()
+ except Exception: # noqa: BLE001
+ return False
+
+ def _the_loader_override_to_blame() -> "str | None":
+ # Which of the three causes it is, since only two are repaired by
+ # installing anything.
+ try:
+ from utils.hardware.amd import the_vulkan_loader_override_to_blame
+ return the_vulkan_loader_override_to_blame()
+ except Exception: # noqa: BLE001
+ return None
+
+ def _reason(text: str) -> str:
+ return f"{text}{_loader_finding}{_second_finding}"
+
+ if node_hint:
+ # A mask hides devices whatever the node permissions are, so a host with
+ # both needs both fixes. Only a mask that can hide EVERY device counts as a
+ # second blocker: a valid selector beside a closed node is not why the probe
+ # came back empty. mask_note below still lists all four -- there it
+ # annotates what torch was looking at rather than claiming a repair.
+ if blocking:
+ node_hint = (
+ f"{node_hint} A device visibility mask is also in force "
+ f"({', '.join(blocking)}), which the groups do not clear."
+ )
+ elif unresolved:
+ node_hint = (
+ f"{node_hint} {', '.join(unresolved)} names a device this cannot "
+ f"resolve, so whether it also hides the card is unknown; check it "
+ f"if the groups do not help."
+ )
+ # A closed node explains an empty probe only when it is one the runtime would
+ # have used. On a multi-AMD host a render node can be shut while a sibling is
+ # open, and the runtime then had a complete path and enumerated nothing
+ # anyway, so the closed one is a SECOND finding. Asked per backend, since HIP
+ # also needs /dev/kfd and that node has no sibling.
+ _node_is_why = (
+ _closed_nodes_block_the_runtime() and not _another_vendor_has_an_open_node()
+ )
+ # A blocker the node repair cannot clear: manifests were found and none
+ # is loadable, so the probe stays empty however the node is owned. Kept out
+ # of node_hint, which may be demoted below: folding it in filed the one
+ # diagnosis that always holds under "not why the probe is empty". "also"
+ # only where the node repair really does precede it.
+ _also = "also " if _node_is_why else ""
+ if _is_vulkan and _the_vulkan_loader_has_no_driver():
+ # The repair depends on WHY. A filter that disables every manifest, and
+ # a forced list pointing at paths that do not resolve, are environment
+ # settings that reinstalling a driver leaves exactly as they were. Named
+ # rather than described, so the user has something to unset.
+ _override = _the_loader_override_to_blame()
+ if _override:
+ _loader_finding = (
+ f" The Vulkan loader {_also}has no driver it can load "
+ f"here, and what leaves it with none is {_override}: clear or "
+ f"correct that, since reinstalling the driver does not change "
+ f"an environment override."
+ )
+ else:
+ _loader_finding = (
+ f" The Vulkan loader {_also}has no driver it can load "
+ f"here: every ICD manifest it would read is missing its library "
+ f"or is 32-bit, so reinstall the Vulkan driver as well."
+ )
+ if _node_is_why:
+ return f"{node_hint}{_loader_finding}"
+ _second_finding = f" Separately, and not why the probe is empty: {node_hint}"
+ if _is_vulkan:
+ return _reason("the Vulkan probe reported no device")
+
try:
import torch
except Exception: # noqa: BLE001
- return f"torch is not importable, so no GPU could be enumerated{mask_note}"
+ return _reason(f"torch is not importable, so no GPU could be enumerated{mask_note}")
if not hasattr(torch, "cuda") or not torch.cuda.is_available():
- return f"torch reports no usable CUDA or HIP device{mask_note}"
+ return _reason(f"torch reports no usable CUDA or HIP device{mask_note}")
# Counting devices does not create a context; reading their memory would.
count = torch.cuda.device_count()
if not count:
- return f"torch enumerated 0 devices{mask_note}"
+ return _reason(f"torch enumerated 0 devices{mask_note}")
if LlamaCppBackend._torch_is_rocm(torch):
coverage = LlamaCppBackend._installed_llama_gfx_archs(binary)
if coverage:
present = sorted(set(LlamaCppBackend._rocm_arch_by_physical_id().values()))
if present and not (set(present) & set(coverage)):
- return (
+ return _reason(
f"the installed llama.cpp build covers {sorted(coverage)} but this "
f"host has {present}, so the arch gate dropped every device"
)
- return (
+ return _reason(
f"torch sees {count} ROCm device(s) but the probe returned none, so "
f"amd-smi and the torch fallback both declined{mask_note}"
)
- return f"torch sees {count} device(s) but the probe returned none{mask_note}"
+ return _reason(f"torch sees {count} device(s) but the probe returned none{mask_note}")
except Exception as e: # noqa: BLE001 -- diagnostics must not break a load
return f"the reason could not be determined ({type(e).__name__})"
diff --git a/studio/backend/tests/test_amd_device_node_permissions.py b/studio/backend/tests/test_amd_device_node_permissions.py
new file mode 100644
index 00000000000..74b885b5267
--- /dev/null
+++ b/studio/backend/tests/test_amd_device_node_permissions.py
@@ -0,0 +1,5187 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""An AMD device node this account cannot open must be named, not read as "no GPU".
+
+Every AMD probe in this tree tests that ``/dev/kfd`` and ``/dev/dri/renderD*`` EXIST.
+On a stock Linux distribution they are ``root:render`` mode 0660, so an account
+outside that group passes all of them and then cannot open the device. HIP counts
+zero devices and the Vulkan loader enumerates none, which is byte-for-byte what
+owning no GPU looks like -- so Studio ran on CPU and told the user to install a
+driver stack that was already loaded (#10466: a fresh Strix Halo install, fixed by
+adding the account to the render and video groups).
+
+Both messages under test used to describe a different host: the capability line
+blamed a PyTorch mismatch and offered Repair installation, and the empty-probe
+explanation blamed the Vulkan probe. Neither repair changes group membership.
+
+No AMD hardware here, and the node paths are patched rather than created: a test
+that had to mknod would need root, which is the one account this bug cannot reach.
+"""
+
+from __future__ import annotations
+
+import builtins
+import getpass
+import inspect
+import io
+import json
+import os
+import platform
+import re
+import shlex
+import subprocess
+import sys
+import types
+from pathlib import Path
+
+
+# grp and pwd are POSIX-only and absent on Windows, and importing them at module level took
+# the ENTIRE file down at collection there: ModuleNotFoundError before a single test ran,
+# so the file reported one collection error instead of its own verdict. The production
+# module imports both INSIDE the functions that need them for the same reason.
+try:
+ import grp
+ import pwd
+except ModuleNotFoundError: # Windows
+ grp = None
+ pwd = None
+
+
+def _running_as_root() -> bool:
+ """Whether this account is root, answered on a platform that has no such idea.
+
+ os.geteuid does not exist on Windows, and these two calls sit in skipif DECORATORS,
+ which are evaluated at import: an AttributeError there is another whole-file collection
+ error rather than a skip.
+ """
+ return getattr(os, "geteuid", lambda: -1)() == 0
+
+
+import pytest
+
+from core.inference.llama_cpp import LlamaCppBackend
+from utils.hardware import amd
+
+# Windows has no /dev, no POSIX groups and no device-node modes, so every subject in this
+# file is absent there rather than merely untestable: amd_nodes_closed_to_this_user answers
+# [] and amd_node_permission_hint answers None on any non-Linux host, which the AMD CI run
+# on a real Windows box confirmed against this very change. macOS is NOT skipped: it has
+# both modules, and the cases that fake a Linux host still exercise the real logic there.
+pytestmark = pytest.mark.skipif(
+ platform.system() == "Windows",
+ reason = (
+ "POSIX-only subject: no /dev device nodes, no grp/pwd, and the probe under test "
+ "returns [] on Windows by construction"
+ ),
+)
+from utils.hardware import hardware
+
+
+_GPU_MASK_VARS = (
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "CUDA_VISIBLE_DEVICES",
+ "GPU_DEVICE_ORDINAL",
+)
+
+
[email protected](autouse = True)
+def _no_inherited_gpu_mask(monkeypatch):
+ """No per-GPU selector unless a test sets one.
+
+ An open render node stops being evidence of a usable path under a mask, so any of these
+ inherited from the runner would decide a case the test never mentioned. This box
+ exports CUDA_VISIBLE_DEVICES, and it silently answered for a control that was supposed
+ to be testing an empty HIP mask. GPU_DEVICE_ORDINAL is here for the same reason and not
+ because any test sets it: a ROCm or OpenCL environment exports it, the production rule
+ reads it, and a test naming no mask would then be answered by the runner's own.
+ """
+ for _var in _GPU_MASK_VARS:
+ monkeypatch.delenv(_var, raising = False)
+
+
[email protected](autouse = True)
+def _the_account_this_process_runs_as(monkeypatch):
+ """The repair commands name the account os.access answered for, so fix what that is.
+
+ A stubbed passwd answer rather than the runner's own, which differs per machine. The
+ environment fallback, for a uid with no passwd entry, has its own test.
+ """
+ # A real struct_passwd, not a SimpleNamespace: getpass.getuser() falls through to
+ # pwd.getpwuid(os.getuid())[0] when none of LOGNAME/USER/LNAME/USERNAME is set, and
+ # pytest calls it while building tmp_path. A non-subscriptable stub raises TypeError
+ # there, which is not among the exceptions pytest catches, so on a runner with no
+ # username in the environment every tmp_path test would die in fixture setup rather
+ # than run.
+ _record = pwd.struct_passwd(("ada", "x", os.getuid(), os.getgid(), "", "/home/ada", "/bin/sh"))
+ monkeypatch.setattr(pwd, "getpwuid", lambda _uid: _record)
+ # ...and the environment fallback agrees with it, so a case that is not about which
+ # account is named does not have to say. The two tests that ARE about it -- a uid with
+ # no passwd entry, and a container whose USER disagrees -- set their own.
+
+
+def linux(monkeypatch):
+ monkeypatch.setattr(amd.platform, "system", lambda: "Linux")
+
+
+# The llama-server that is never there: every empty-probe case below asks about an install
+# whose binary does not exist, because the reason under test is decided before it is run.
+_NO_SUCH_SERVER = "/nonexistent/llama-server"
+
+# The two device nodes a ROCm host has: the KFD, which only HIP opens, and one render
+# node, which HIP and the Vulkan loader both open. Named because most cases below are
+# about one of the two being shut while the other is not.
+_AMD_NODES = ["/dev/kfd", "/dev/dri/renderD128"]
+
+# The order _groups_that_own returns its buckets in. Named here so a case can say which
+# bucket it is about instead of counting commas in an eight-element tuple.
+_BUCKETS = (
+ "joinable",
+ "unnamed",
+ "no_group",
+ "acl",
+ "owned",
+ "privileged",
+ "already",
+ "external",
+)
+
+
+def _buckets(**named) -> tuple:
+ """The tuple ``_groups_that_own`` returns, naming only the buckets a case populates."""
+ assert not set(named) - set(_BUCKETS), f"unknown bucket: {set(named) - set(_BUCKETS)}"
+ return tuple(list(named.get(_name, [])) for _name in _BUCKETS)
+
+
+# The real derivation, captured before any case stubs it. _nodes() stubs _groups_that_own
+# for every case that is not about it, so a case that IS about it has to put the real one
+# back rather than describe the host twice.
+_REAL_GROUPS_THAT_OWN = amd._groups_that_own
+
+
+def _the_real_group_derivation(monkeypatch):
+ """Undo _nodes()'s stub, for a case whose point is what the nodes themselves say."""
+ monkeypatch.setattr(amd, "_groups_that_own", _REAL_GROUPS_THAT_OWN)
+
+
+def _owning(monkeypatch, **named):
+ """Stub the group derivation with the buckets this case is about.
+
+ Every arm that is not testing the derivation itself stubs it, because these node paths
+ are patched rather than created and stat cannot name their groups. Saying so explicitly
+ keeps the answer off whatever the runner happens to have at the same path.
+ """
+ _answer = _buckets(**named)
+ monkeypatch.setattr(amd, "_groups_that_own", lambda paths: _answer)
+
+
+def _ggml(monkeypatch, backends):
+ """The ggml libraries this llama.cpp install ships, which decides whose node it opens."""
+ monkeypatch.setattr(
+ LlamaCppBackend, "_installed_ggml_backends", staticmethod(lambda _b: frozenset(backends))
+ )
+
+
+def _empty_probe() -> str:
+ """The reason ``_explain_empty_gpu_probe`` gives for a GPU probe that found nothing."""
+ return LlamaCppBackend._explain_empty_gpu_probe(_NO_SUCH_SERVER)
+
+
+def _capability_message(verdict: str, detail: "str | None" = None) -> str:
+ """The line Studio shows for a GPU it can see and cannot use, for a given verdict."""
+ return hardware._gpu_present_but_unusable_message(
+ "video generation",
+ verdict = (verdict, detail),
+ )
+
+
+def _mismatch_vendors(monkeypatch, vendors):
+ """Which vendors qualified for the capability mismatch, which is whose card raised it."""
+ monkeypatch.setattr(hardware, "CHAT_ONLY_MISMATCH_VENDORS", frozenset(vendors))
+
+
+def _nodes(
+ monkeypatch,
+ *,
+ present: list[str],
+ openable: set[str],
+ amd_owned: bool = True,
+ vendor_readable: bool = True,
+ topology: "bool | None | str" = "as-owned",
+ gpu_count: "int | None | str" = "as-present",
+):
+ """A host whose ``present`` nodes exist and whose ``openable`` subset can be opened.
+
+ ``amd_owned`` is the vendor of the hardware behind those nodes, stubbed here and
+ exercised for real in the two tests below it. ``vendor_readable`` is whether sysfs will
+ say so: a container can map the node and hide the entry that names its vendor, and the
+ two are different answers.
+
+ ``gpu_count`` is how many AMD GPU agents KFD enumerates, and it defaults to the AMD
+ render nodes this host was given, since that is what the described host would report.
+ It is stubbed here rather than left live for the reason this whole helper exists: it
+ reads /sys/class/kfd, so a case that merely sets a visibility mask was asking the
+ RUNNER how many GPUs it has. On a machine with no AMD card that read is None, which
+ means "no bound", so every selector looks like a narrowing; on a real gfx1151 it is 1,
+ so a selector naming device 0 looks like the whole host. Those are opposite verdicts,
+ and that is precisely how this file passed here and failed on the hardware it is
+ written for. Pass None explicitly for the unreadable-topology case.
+ """
+ # Only the device-node enumeration. The module's other caller of the same helper walks
+ # the Vulkan icd.d directories, and answering that one with a list of render nodes made
+ # every loader question read as "no drivers at all" -- which is a verdict, not an
+ # absence, so it would have passed silently.
+ _real_glob = amd.glob.glob
+ monkeypatch.setattr(
+ amd.glob,
+ "glob",
+ lambda pattern: (
+ [p for p in present if p.startswith("/dev/dri/renderD")]
+ if pattern.startswith("/dev/dri/")
+ else _real_glob(pattern)
+ ),
+ )
+ monkeypatch.setattr(amd.os.path, "exists", lambda p: p in present)
+ monkeypatch.setattr(amd.os, "access", lambda p, mode: p in openable)
+ monkeypatch.setattr(amd, "_render_node_is_amd", lambda p: vendor_readable and amd_owned)
+ # The vendor, not the verdict: _amd_render_node_exists reads it directly, so that an
+ # unreadable entry can be told apart from one that named another vendor.
+ monkeypatch.setattr(
+ amd,
+ "_render_node_vendor",
+ lambda p: None if not vendor_readable else ("0x1002" if amd_owned else "0x10de"),
+ )
+ # None is a topology that could not be READ, which the closed-node walk answers
+ # differently from one that read and named another vendor. Defaults to the hardware so
+ # every arm written before the distinction existed is unaffected.
+ _topology = amd_owned if topology == "as-owned" else topology
+ monkeypatch.setattr(amd, "_kfd_topology_has_an_amd_gpu", lambda: _topology is True)
+ monkeypatch.setattr(amd, "_kfd_topology_amd_state", lambda: _topology)
+ if gpu_count == "as-present":
+ _renders = [p for p in present if p.startswith("/dev/dri/renderD")]
+ # Zero is not a bound, and the production reader says so too: a topology naming no
+ # GPU bounds nothing, so it answers None rather than 0.
+ gpu_count = len(_renders) if (_renders and amd_owned) else None
+ monkeypatch.setattr(amd, "amd_kfd_gpu_node_count", lambda: gpu_count)
+ # These paths are patched rather than created, so stat cannot name their groups; say
+ # so explicitly instead of leaving it to whether the runner happens to have a node at
+ # the same path. The derivation itself is exercised in its own tests below.
+ _owning(monkeypatch)
+
+
+# The parameter tables below are hand-wrapped, one case to a row. Left to the formatter
+# each row becomes one argument per line, which is most of the length the review objected
+# to, so this span is fenced off. Nothing but the wrapping depends on the fence.
+# fmt: off
+# The node layouts the families below are written over, named once so a case can say which
+# host it is about. SHUT is present and refused, OPEN is present and openable.
+_KFD_SHUT = dict(present = ["/dev/kfd"], openable = set())
+_RENDER_SHUT = dict(present = ["/dev/dri/renderD128"], openable = set())
+_BOTH_SHUT = dict(present = _AMD_NODES, openable = set())
+_BOTH_OPEN = dict(present = _AMD_NODES, openable = set(_AMD_NODES))
+_KFD_OPEN = dict(present = ["/dev/kfd"], openable = {"/dev/kfd"})
+_RENDER_OPEN = dict(present = ["/dev/dri/renderD128"], openable = {"/dev/dri/renderD128"})
+_ONLY_KFD_SHUT = dict(present = _AMD_NODES, openable = {"/dev/dri/renderD128"})
+_NO_NODES = dict(present = [], openable = set())
+_NVIDIA_RENDER_SHUT = dict(present = ["/dev/dri/renderD128"], openable = set(), amd_owned = False)
+_NVIDIA_KFD_OPEN = dict(present = ["/dev/kfd"], openable = {"/dev/kfd"}, amd_owned = False)
+
+
+ pytest.param(("Linux", _BOTH_SHUT, _AMD_NODES), id = "a_node_this_user_cannot_open"),
+ # The control. Without it every assertion here also passes on a host with no AMD
+ # hardware, where the list is empty for a reason that is not this bug.
+ pytest.param(("Linux", _BOTH_OPEN, []), id = "a_host_whose_nodes_open"),
+ pytest.param(("Linux", _NO_NODES, []), id = "a_host_with_no_amd_nodes"),
+ # The false positive this nearly shipped with: render nodes are root:render for EVERY
+ # vendor, so this box (8 NVIDIA cards, an account outside the render group) listed
+ # all eight and sent a CUDA user after the AMD groups. CUDA opens /dev/nvidia*.
+ pytest.param(("Linux", _NVIDIA_RENDER_SHUT, []), id = "an_nvidia_hosts_closed_render_node"),
+ # macOS and Windows have no render nodes, and os.access on Windows answers for a
+ # permission model this message does not describe.
+ pytest.param(("Windows", _KFD_SHUT, []), id = "the_probe_is_linux_only"),
+])
+def test_which_nodes_are_reported_closed(monkeypatch, case):
+ """The closed list itself. What these hosts are then TOLD is the two families below."""
+ system, layout, closed = case
+ monkeypatch.setattr(amd.platform, "system", lambda: system)
+ _nodes(monkeypatch, **layout)
+ assert amd.amd_nodes_closed_to_this_user() == closed
+
+
+def test_the_vendor_is_read_from_sysfs(monkeypatch):
+ """The reader the cases above stub. sysfs is world-readable, so ownership is answerable
+ without the access being tested for."""
+ real_open = builtins.open
+
+ def _fake(path, *a, **k):
+ if str(path) == "/sys/class/drm/renderD128/device/vendor":
+ return io.StringIO("0x1002\n")
+ if str(path) == "/sys/class/drm/renderD129/device/vendor":
+ return io.StringIO("0x10de\n")
+ return real_open(path, *a, **k)
+
+ monkeypatch.setattr(builtins, "open", _fake)
+ assert amd._render_node_is_amd("/dev/dri/renderD128") is True
+ assert amd._render_node_is_amd("/dev/dri/renderD129") is False
+ assert amd._render_node_is_amd("/dev/dri/renderD130") is False # unreadable
+
+
+def test_a_node_that_cannot_be_stat_ed_is_skipped(monkeypatch, linux):
+ """A probe that raises must not break a load; the caller is a diagnostic."""
+
+ def _boom(_p):
+ raise OSError("stale handle")
+
+ monkeypatch.setattr(amd.glob, "glob", lambda pattern: [])
+ monkeypatch.setattr(amd.os.path, "exists", _boom)
+ assert amd.amd_nodes_closed_to_this_user() == []
+
+
+def test_read_only_access_is_not_enough(monkeypatch, linux):
+ """HIP and the Vulkan loader both open the node read-write, so a node that only reads is
+ still unusable and answering "fine" here would restore the silence."""
+ _nodes(monkeypatch, **_KFD_SHUT)
+ monkeypatch.setattr(amd.os, "access", lambda p, mode: mode == os.R_OK)
+ assert amd.amd_nodes_closed_to_this_user() == ["/dev/kfd"]
+
+
+_USERMOD = "usermod -a -G render,video ada"
+
+
+def _asserts(text: str, says, does_not_say):
+ """Every ``says`` is in ``text`` and every ``does_not_say`` is not. Both directions run:
+ a family that asserted only the positive half would pass on a message that also said the
+ thing it is meant to have stopped saying."""
+ for _wanted in says:
+ assert _wanted in text
+ for _unwanted in does_not_say:
+ assert _unwanted not in text
+
+
+ pytest.param((_KFD_SHUT, {}, True, ("/dev/kfd", _USERMOD), ()),
+ id = "the_hint_names_the_nodes_the_groups_and_the_account"),
+ # Vulkan never opens /dev/kfd, so a closed one does not stop every backend. Claiming
+ # it did sent a Vulkan user with an unrelated failure after ROCm groups.
+ pytest.param((_KFD_SHUT, {}, True, ("ROCm cannot use",), ("no GPU backend",)),
+ id = "the_hint_is_rocm_specific_when_only_kfd_is_closed"),
+ # Its pair: HIP and the Vulkan loader both open the render node.
+ pytest.param((_RENDER_SHUT, {}, True, ("no GPU backend can use",), ()),
+ id = "the_hint_covers_every_backend_when_a_render_node_is_closed"),
+ # No membership creates /dev/kfd, so a ROCm caller needs both sentences. The
+ # kernel-stack wording this used to assert was wrong, and asserting it is what kept
+ # it: that sentence needs the KFD topology, which is amdkfd's own sysfs, so the stack
+ # is loaded wherever it is readable. install.sh's branch is gated on it being ABSENT.
+ pytest.param((_RENDER_SHUT, {}, True, (_USERMOD, "/dev/kfd"), ("kernel stack",)),
+ id = "the_hint_says_so_when_kfd_does_not_exist_at_all"),
+ # The control: /dev/kfd exists, so the stack is loaded and the groups are the whole
+ # repair. Telling this user to install one would be the #10466 mistake.
+ pytest.param((_BOTH_SHUT, {}, True, (), ("kernel stack",)),
+ id = "a_closed_but_present_kfd_node_says_nothing_about_the_kernel_stack"),
+ # And Vulkan never opens /dev/kfd, so its absence is not that caller's problem.
+ pytest.param((_RENDER_SHUT, {}, False, (), ("kernel stack",)),
+ id = "a_vulkan_caller_is_not_told_about_a_kernel_stack_it_does_not_need"),
+ # render,video is the usual pair, not a universal truth: a container gets numeric gids
+ # with no matching NAMES, a minimal distribution can ship no render group, and a
+ # root:root node is not fixed by joining. Fails before the fix, which hard-coded it.
+ pytest.param((_BOTH_SHUT, dict(joinable = ["kfd", "gpu"]), True, ("usermod -a -G kfd,gpu ada",),
+ ("render,video",)),
+ id = "the_repair_names_the_groups_the_closed_nodes_belong_to"),
+ # One group named, so the sentence has to agree with the command rather than saying
+ # "groups" over a single name.
+ pytest.param((_RENDER_SHUT, dict(joinable = ["render"]), True,
+ ("usermod -a -G render ada", "render group and then log out"), ()),
+ id = "a_single_owning_group_is_not_pluralised"),
+ # The control: the derivation is best effort, so a host whose nodes cannot be stat'd
+ # still gets advice rather than an empty -G argument, and that is the documented pair.
+ pytest.param((_KFD_SHUT, {}, True, (_USERMOD,), ()),
+ id = "unreadable_nodes_fall_back_to_the_documented_pair"),
+ # usermod -a -G takes names only: shadow 4.13 answers ``group '993' does not exist``
+ # and exits 6, run live here rather than read out of the man page. Fails before the
+ # fix, which put the bare number in -G. The assertion is on the NUMBER, not on the
+ # command, because the repair does name usermod after a groupadd that names the GID.
+ pytest.param((_KFD_SHUT, dict(unnamed = [993]), True, ("--group-add 993",),
+ ("usermod -a -G 993",)),
+ id = "an_unnamed_gid_is_not_handed_to_usermod"),
+ # The control that keeps that suppression narrow: a host with one node in a real
+ # group and another in an unnamed one can still fix half of it by joining.
+ pytest.param((_BOTH_SHUT, dict(joinable = ["render"], unnamed = [993]), True,
+ ("usermod -a -G render ada", "GID 993"), ()),
+ id = "a_joinable_group_beside_an_unnamed_gid_is_still_prescribed"),
+ # A udev rule leaving the node root:render 0600 denies its own group, so joining
+ # render runs, succeeds, and opens nothing: the repair there is the rule. Fails before
+ # the fix, which named the owning group whatever the mode said.
+ pytest.param((_KFD_SHUT, dict(no_group = ["/dev/kfd"]), True, ("udev rule",),
+ ("usermod -a -G",)),
+ id = "a_node_no_membership_opens_is_not_answered_with_usermod"),
+ # The control for BOTH suppressions: every list empty means the nodes could not be
+ # stat'd at all, a detection miss rather than evidence that joining cannot work.
+ pytest.param((_KFD_SHUT, {}, True, (_USERMOD,), ()),
+ id = "a_host_whose_nodes_could_not_be_read_still_gets_the_documented_pair"),
+ # /dev/kfd without /dev/dri passes every probe here and still cannot initialise ROCm,
+ # because ROCr opens a render node to reach amdgpu, which is why docker/run.sh passes
+ # both. No group creates the missing one. Fails before the fix, which named only kfd.
+ pytest.param((_KFD_SHUT, {}, True,
+ ("No AMD render node", "--device /dev/kfd --device /dev/dri"), ()),
+ id = "a_container_given_kfd_but_no_render_node_is_told_so"),
+ # Its control: an ordinary AMD host has a render node and only cannot open it, so
+ # claiming the device mapping is wrong invents a second repair.
+ pytest.param((_BOTH_SHUT, {}, True, (), ("No AMD render node",)),
+ id = "a_host_that_has_a_render_node_is_not_told_to_map_one"),
+ # The sentence for the external-denial bucket, since the bucket cases below only
+ # decide it: naming a mode to fix on a node whose mode already grants rw cannot work.
+ pytest.param((_KFD_SHUT, dict(external = ["/dev/kfd"]), True,
+ ("granted read and write by the permission bits that apply to this account",
+ "container device cgroup or an LSM"),
+ # Not "owner": this bucket also holds a node the account neither owns nor
+ # shares a group with, whose OTHER bits grant rw, and claiming ownership
+ # there is a statement the reader can check and find false.
+ ("chmod", "is owned by this account")),
+ id = "the_external_denial_sentence_does_not_prescribe_a_mode_change"),
+ # docker's --group-add takes ONE value, so two unnamed groups need the flag twice.
+ # Fails before the fix, which interpolated unnamed[0] alone.
+ pytest.param((_BOTH_SHUT, dict(unnamed = [993, 994]), True,
+ ("--group-add 993 --group-add 994", "GIDs 993, 994"), ()),
+ id = "every_unnamed_gid_reaches_the_docker_repair"),
+ # The control on the wording: the one-GID host is the common one.
+ pytest.param((_KFD_SHUT, dict(unnamed = [993]), True,
+ ("GID 993, which has", "--group-add 993."), ()),
+ id = "a_lone_unnamed_gid_is_still_named_in_the_singular"),
+ # root:root 0660 opens for anyone in the root group, so the derivation would print
+ # `sudo usermod -a -G root`. That grants far more than the GPU: a udev
+ # misconfiguration to report rather than a repair.
+ pytest.param((_KFD_SHUT, dict(privileged = ["root"]), True, ("root", "udev"), ("usermod",)),
+ id = "the_hint_for_a_privileged_owner_says_it_is_not_the_repair"),
+ # The same mapping with that node OPEN: the closed list is empty, and this returned
+ # None while ROCr still had no render node. A missing node is not a permission
+ # problem, so the hint cannot be gated on one.
+ pytest.param((_KFD_OPEN, {}, True, ("AMD render node", "--device /dev/dri"), ("usermod",)),
+ id = "a_container_with_an_open_kfd_and_no_render_node_is_still_told"),
+ # The mirror image, --device /dev/dri alone: `closed` is empty and `_render_missing`
+ # false, so the hint returned None before its own missing-KFD sentence and left the
+ # caller on reinstall advice.
+ pytest.param((_RENDER_OPEN, {}, True, ("/dev/kfd",), ("usermod",)),
+ id = "a_container_given_only_the_render_node_is_told_about_kfd"),
+])
+def test_what_the_hint_says_about_a_host(monkeypatch, linux, case):
+ """The sentence a user reads, per host and per owning-group derivation. ``owning`` stubs
+ _groups_that_own, which _nodes defaults to the empty answer; the derivation itself is
+ exercised against real modes further down."""
+ layout, owning, needs_kfd, says, does_not_say = case
+ _nodes(monkeypatch, **layout)
+ _owning(monkeypatch, **owning)
+ hint = amd.amd_node_permission_hint(needs_kfd = needs_kfd)
+ assert hint is not None
+ _asserts(hint, says, does_not_say)
+
+
+ # The control for the closed list above: a host whose nodes open is told nothing, so
+ # the assertions here cannot be passing for the absence of hardware.
+ pytest.param((_BOTH_OPEN, True, False), id = "a_host_whose_nodes_open_is_told_nothing"),
+ # And an NVIDIA host's closed render node is not this bug, whatever its mode says.
+ pytest.param((_NVIDIA_RENDER_SHUT, True, False), id = "an_nvidia_host_is_told_nothing"),
+ # ``needs_kfd = False`` is a Vulkan binary saying a closed KFD node is not its
+ # problem. The render node is open, so the only thing wrong is the node that caller
+ # disclaimed; without an open one the answer would be a real Vulkan blocker.
+ pytest.param((_ONLY_KFD_SHUT, False, False),
+ id = "a_vulkan_only_caller_is_not_answered_with_a_closed_kfd_node"),
+ # The same host asked by a caller that DOES need the KFD node still gets an answer,
+ # or the narrowing above has simply silenced the hint.
+ pytest.param((_ONLY_KFD_SHUT, True, True), id = "a_rocm_caller_on_that_host_is_answered"),
+ # The other half of that control, so the narrowing cannot silence the real case.
+ pytest.param((_RENDER_SHUT, False, True),
+ id = "a_vulkan_caller_is_still_answered_about_a_closed_render_node"),
+ # The trap a bare "the glob is empty" test falls into: every vendor's render nodes
+ # live under /dev/dri/renderD*, so the AMD signal must survive having none. The KFD
+ # topology names the vendor and is world-readable.
+ pytest.param((_NVIDIA_KFD_OPEN, True, False),
+ id = "a_host_with_no_amd_card_is_not_told_to_map_a_render_node"),
+ # And the reason needs_kfd exists: a Vulkan failure with some other cause must not be
+ # sent after the ROCm kernel stack.
+ pytest.param((_RENDER_OPEN, False, False),
+ id = "the_same_mapping_says_nothing_to_a_vulkan_caller"),
+])
+def test_whether_the_hint_answers_at_all(monkeypatch, linux, case):
+ """Which hosts get a hint at all, before asking what it says."""
+ layout, needs_kfd, answered = case
+ _nodes(monkeypatch, **layout)
+ assert (amd.amd_node_permission_hint(needs_kfd = needs_kfd) is not None) is answered
+
+
+def _no_passwd_entry(monkeypatch):
+ """A uid the passwd database does not know, which is where the environment is read."""
+
+ def _missing(_uid):
+ raise KeyError(_uid)
+
+ monkeypatch.setattr(pwd, "getpwuid", _missing)
+
+
+ # `docker run --user 1234` leaves the uid with no passwd entry while USER commonly
+ # still says root. usermod against that name succeeds, changes an identity nothing is
+ # running as, and leaves the nodes shut: the repair is the container's group wiring.
+ # Fails before the fix, which fell back to USER and then to a literal $USER.
+ pytest.param((dict(joinable = ["render"]), None, ("--group-add render",),
+ ("usermod -a -G", "root")),
+ id = "a_uid_with_no_passwd_entry_is_not_given_a_usermod"),
+ # The control: a uid with a passwd entry is an account usermod can name, and that is
+ # the repair on every ordinary host. Without it the fix could be "never prescribe
+ # usermod", which removes what #10466 asked for.
+ pytest.param((dict(joinable = ["render"]), "ada", ("sudo usermod -a -G render ada",),
+ ("--group-add",)),
+ id = "an_account_the_system_knows_still_gets_the_command"),
+ # The unnamed-GID repair is a groupadd AND a usermod, and the second half needs the
+ # same account, so printing the pair here would be two commands that cannot both work.
+ pytest.param((dict(unnamed = [993]), None, ("--group-add 993",),
+ ("usermod -a -G", "groupadd -g")),
+ id = "an_unnamed_gid_under_that_uid_drops_the_groupadd_half_too"),
+])
+def test_which_account_the_repair_names(monkeypatch, linux, case):
+ """Whether the repair can name an account at all. USER is root in every arm, so a command
+ that names it is one that read the environment instead of the passwd entry."""
+ owning, account, says, does_not_say = case
+ _nodes(monkeypatch, **_KFD_SHUT)
+ monkeypatch.setenv("USER", "root")
+ _owning(monkeypatch, **owning)
+ if account is None:
+ _no_passwd_entry(monkeypatch)
+ else:
+ monkeypatch.setattr(amd, "_repair_account", lambda: account)
+ hint = amd.amd_node_permission_hint()
+ _asserts(hint, says, does_not_say)
+
+
+def _pins(
+ rocm_intent = None,
+ hip_runtime = None,
+ other_vendor = None,
+) -> dict:
+ """Which of the three runtime probes a capability case pins, and to what.
+
+ Pinned rather than inherited: all three read whatever torch is installed beside the test,
+ and this workspace carries a ROCm one, so a live probe makes an arm pass or fail on the
+ runner's wheel rather than on the host the case describes.
+ """
+ _named = {
+ "_expected_rocm_flavor_was_chosen": rocm_intent,
+ "_torch_reports_a_hip_runtime": hip_runtime,
+ "_torch_reports_another_vendors_runtime": other_vendor,
+ }
+ return {_probe: _answer for _probe, _answer in _named.items() if _answer is not None}
+
+
+ # Fails before the fix: the old message blamed PyTorch and offered Repair
+ # installation, which cannot add an account to a group.
+ pytest.param((_KFD_SHUT, {"amd"}, "torch_cuda_unavailable", "2.11.0+rocm7.0", _pins(),
+ (_USERMOD,), ("Repair installation",)),
+ id = "the_capability_message_names_the_permission_not_a_torch_mismatch"),
+ # The control: on any other unusable-GPU host the PyTorch wording has to survive, or
+ # this fix trades one wrong answer for another. A render node too, and open: "the
+ # nodes open" must mean every node this host needs, or the control is a container
+ # missing /dev/dri getting a message about that instead.
+ pytest.param((_BOTH_OPEN, {"amd"}, "torch_cuda_unavailable", "2.11.0+rocm7.0", _pins(),
+ ("Repair installation",), ("usermod",)),
+ id = "the_capability_message_is_unchanged_when_the_nodes_open"),
+ # A hybrid host whose NVIDIA card raised the verdict while an AMD node happens to be
+ # closed. Joining the render group repairs nothing there and reinstalling might.
+ pytest.param((_KFD_SHUT, {"nvidia"}, "torch_cuda_unavailable", "2.11.0+cu130", _pins(),
+ ("Repair installation",), ("usermod",)),
+ id = "an_nvidia_mismatch_keeps_the_pytorch_message"),
+ # Its pair, one recorded vendor apart: on a hybrid host the hint needs the install to
+ # target AMD, and a venv that asked for ROCm and got a CPU wheel is this verdict.
+ pytest.param((_KFD_SHUT, {"amd", "nvidia"}, "torch_cpu_build", None,
+ _pins(rocm_intent = True, hip_runtime = False, other_vendor = False), (_USERMOD,),
+ ()),
+ id = "a_hybrid_host_whose_amd_card_raised_it_still_gets_the_hint"),
+ # Opening the node leaves a CPU-only wheel with no GPU path, so the reinstall step has
+ # to survive the permission hint rather than be replaced by it.
+ pytest.param((_KFD_SHUT, {"amd"}, "torch_cpu_build", None, _pins(),
+ ("CPU-only build", "Repair installation", _USERMOD), ()),
+ id = "a_cpu_wheel_beside_a_closed_node_is_told_to_do_both"),
+ # The pair: a ROCm wheel that cannot open a device is fully explained by the node.
+ pytest.param((_KFD_SHUT, {"amd"}, "torch_cuda_unavailable", "2.11.0+rocm7.0", _pins(),
+ (_USERMOD,), ("Repair installation",)),
+ id = "a_gpu_wheel_beside_a_closed_node_is_told_only_the_permission"),
+ # A supported AMD card qualifies for the mismatch whatever wheel is installed, so a
+ # hybrid host records both vendors even when the verdict is about the NVIDIA card. No
+ # membership makes a CUDA wheel use the AMD card.
+ pytest.param((_KFD_SHUT, {"amd", "nvidia"}, "torch_cuda_unavailable", "2.11.0+cu130",
+ _pins(rocm_intent = False, hip_runtime = False), ("Repair installation",),
+ ("usermod",)),
+ id = "a_hybrid_host_running_cuda_torch_keeps_the_pytorch_message"),
+ # The control, and the #10466 host: AMD is the only vendor that qualified, so the
+ # verdict can only be about it and the wheel's own tag adds nothing.
+ pytest.param((_KFD_SHUT, {"amd"}, "torch_cuda_unavailable", "2.11.0+rocm7.0",
+ _pins(rocm_intent = False, hip_runtime = False), (_USERMOD,), ()),
+ id = "an_amd_only_host_needs_no_runtime_evidence"),
+ # An AMD-only host running a CUDA-tagged wheel raises the same verdict, and opening
+ # the node does not make that wheel use the card, so it needs both. Fails before the
+ # fix, where "AMD is the only vendor" made the hint REPLACE the reinstall advice and
+ # left the user with a group command and no way to use the GPU.
+ pytest.param((_KFD_SHUT, {"amd"}, "torch_cuda_unavailable", "2.11.0+cu128",
+ _pins(rocm_intent = False, hip_runtime = False),
+ (_USERMOD, "Repair installation"), ()),
+ id = "a_cuda_wheel_on_an_amd_only_host_keeps_the_reinstall_advice"),
+ # Its control, one label apart: a ROCm wheel that cannot initialise the device IS
+ # fully explained by the closed node, so the reinstall advice stays suppressed.
+ pytest.param((_KFD_SHUT, {"amd"}, "torch_cuda_unavailable", "2.11.0+rocm7.0",
+ _pins(rocm_intent = False, hip_runtime = False), (_USERMOD,),
+ ("Repair installation",)),
+ id = "a_rocm_wheel_on_the_same_host_still_replaces_it"),
+ # A venv that recorded ROCm intent and then had a CUDA build installed over it still
+ # answers yes to _expected_rocm_flavor_was_chosen, and reading that as "the wheel
+ # targets AMD" replaced the repair this host needs with a sentence about groups. The
+ # hint is APPENDED instead: the node is real, and the wheel is still the repair.
+ pytest.param((_BOTH_SHUT, {"amd"}, "torch_cuda_unavailable", "2.11.0+cu128",
+ _pins(rocm_intent = True, hip_runtime = False),
+ ("matching PyTorch build fixes it", "cannot open"), ()),
+ id = "a_wheel_tagged_for_another_vendor_keeps_the_reinstall_advice"),
+ # The control: +cpu names no accelerator, so it settles nothing about which vendor
+ # the install targets and the recorded intent is the best evidence there is. Without
+ # it the fix reads as "any non-ROCm label wins", which silences the hint on the
+ # CPU-torch host #10466 was reported from.
+ pytest.param((_BOTH_SHUT, {"amd", "nvidia"}, "torch_cpu_build", "2.11.0+cpu",
+ _pins(rocm_intent = True, hip_runtime = False, other_vendor = False),
+ ("cannot open",), ()),
+ id = "a_label_that_names_no_vendor_still_lets_the_intent_speak"),
+])
+def test_the_capability_message_for_a_host(monkeypatch, linux, case):
+ """The line Studio shows for a GPU it can see and cannot use."""
+ layout, vendors, verdict, detail, pins, says, does_not_say = case
+ _nodes(monkeypatch, **layout)
+ _mismatch_vendors(monkeypatch, vendors)
+ for _probe, _answer in pins.items():
+ monkeypatch.setattr(hardware, _probe, lambda _a = _answer: _a)
+ message = _capability_message(verdict, detail)
+ _asserts(message, says, does_not_say)
+
+
+ # The load-time line. It runs ahead of the Vulkan branch on purpose: a closed render
+ # node is the reason UNDERNEATH "the Vulkan probe reported no device", and that
+ # phrasing sends the user after a driver that is already working.
+ pytest.param((_RENDER_SHUT, {"vulkan"}, {}, (_USERMOD,),
+ ("the Vulkan probe reported no device",)),
+ id = "the_empty_probe_explanation_names_the_permission"),
+ # The control for that narrowing: same host, a non-Vulkan binary, hint must survive.
+ pytest.param((_ONLY_KFD_SHUT, {"hip"}, {}, (_USERMOD,), ()),
+ id = "a_rocm_binary_is_still_told_about_the_closed_kfd_node"),
+ # A hybrid host whose CUDA build enumerated nothing for its own reasons. Every AMD
+ # node is closed, so the hint is available and any build that could use the card gets
+ # it. This one cannot, so the mask diagnosis has to survive.
+ pytest.param((_BOTH_SHUT, {"cuda"}, {"CUDA_VISIBLE_DEVICES": ""}, ("CUDA_VISIBLE_DEVICES",),
+ ("usermod",)),
+ id = "a_cuda_only_build_is_not_sent_after_the_amd_render_group"),
+ # Its control, and why the rule names CUDA rather than "not ROCm": an install this
+ # probe cannot read must not lose the diagnosis.
+ pytest.param((_BOTH_SHUT, set(), {}, (_USERMOD,), ()),
+ id = "a_build_whose_backend_cannot_be_read_still_gets_the_hint"),
+ # _is_vulkan_backend defers such a build to CUDA, so it is a CUDA install here too.
+ # Requiring CUDA to be the ONLY shipped library left this layout uncovered.
+ pytest.param((_BOTH_SHUT, {"cuda", "vulkan"}, {"CUDA_VISIBLE_DEVICES": ""},
+ ("CUDA_VISIBLE_DEVICES",), ("usermod",)),
+ id = "a_cuda_plus_vulkan_build_is_treated_as_cuda"),
+ # A build with no GPU library cannot offload to any card, so its empty probe is not a
+ # permission problem and the groups cannot change it.
+ pytest.param((_BOTH_SHUT, {"cpu", "base"}, {}, (), ("usermod",)),
+ id = "a_cpu_only_llama_build_is_not_sent_after_the_groups"),
+ # Two independent blockers need two fixes. The groups do not clear a visibility mask,
+ # so returning the hint alone hid the half the user also has to undo.
+ pytest.param((_BOTH_SHUT, {"hip"}, {"HIP_VISIBLE_DEVICES": "-1"},
+ (_USERMOD, "HIP_VISIBLE_DEVICES='-1'"), ()),
+ id = "a_mask_is_reported_alongside_the_permission_hint"),
+])
+def test_the_empty_probe_reason_for_an_install(monkeypatch, linux, case):
+ """Which reason the load-time explanation gives, per install and node layout."""
+ layout, backends, env, says, does_not_say = case
+ _nodes(monkeypatch, **layout)
+ for _var, _value in env.items():
+ monkeypatch.setenv(_var, _value)
+ _ggml(monkeypatch, backends)
+ reason = _empty_probe()
+ _asserts(reason, says, does_not_say)
+
+
[email protected]("layout", [
+ pytest.param(_RENDER_OPEN, id = "the_explanation_is_unchanged_when_the_nodes_open"),
+ # End to end through the caller: a Vulkan binary on a host whose render node opens
+ # must not be told about ROCm's node.
+ pytest.param(_ONLY_KFD_SHUT, id = "the_vulkan_reason_survives_a_closed_kfd_node"),
+])
+def test_a_vulkan_host_whose_own_node_opens_keeps_its_own_reason(monkeypatch, linux, layout):
+ """The control for the family above: a Vulkan host that can open what Vulkan uses keeps
+ its own reason, whole and unqualified."""
+ _nodes(monkeypatch, **layout)
+ _ggml(monkeypatch, {"vulkan"})
+ assert _empty_probe() == "the Vulkan probe reported no device"
+
+
+def test_no_mask_leaves_the_hint_alone(monkeypatch, linux):
+ """The control: the sentence must not grow a trailing clause on a host with no mask set,
+ which is every host the #10466 wording was written for."""
+ _nodes(monkeypatch, **_BOTH_SHUT)
+ for var in _GPU_MASK_VARS:
+ monkeypatch.delenv(var, raising = False)
+ _ggml(monkeypatch, {"hip"})
+ reason = _empty_probe()
+ assert reason.endswith("sudo usermod -a -G render,video ada")
+
+
+def _kernel_stack_hint_runs(
+ tmp_path,
+ closed_nodes: str,
+ *,
+ route: bool = True,
+ nvidia: bool = False,
+) -> bool:
+ """Whether install.sh's missing-kernel-stack branch fires for this closed set.
+
+ The guard is lifted by text, not restated: a restatement would pass whatever the
+ installer went on to say. Only the condition is taken, and its two probes are stubbed
+ true so the answer depends on nothing but the closed-node reasoning.
+ """
+ lines = _install_sh_lines()
+ # Anchored on the part of the condition this change does NOT touch, then walked back to
+ # the "if". Anchoring on the new closed-node text would make the control vacuous: a
+ # revert would stop the extraction finding anything, and "the text changed" would read
+ # as "the behaviour changed".
+ end = _install_sh_anchor(lines, _PCI_SENTENCE)
+ start = _install_sh_if_above(lines, end)
+ guard = "\n".join(line.strip() for line in lines[start : end + 1])
+ script = "\n".join(
+ [
+ "_has_amd_rocm_gpu() { return 1; }", # ROCm cannot see the card
+ "_amd_gpu_present_via_pci() { return 0; }", # but the PCI bus can
+ # Carried by the guard now that it sits after the case rather than inside the
+ # */cpu arm, which supplied them.
+ "SKIP_TORCH=false",
+ "OS=linux",
+ # The run-scope predicate the guard asks in place of a bare SKIP_TORCH test.
+ # Lifted, not stubbed, so this arm goes through the installer's own rule.
+ *_run_scope_defs(lines, nvidia = nvidia),
+ # The route gate, true by default: this harness asks about the closed-node
+ # reasoning, and the route has its own tests below.
+ f"_amd_node_diag_route={'true' if route else 'false'}",
+ # The guard prints through substep on the way to the arm under test, and the
+ # chain asks the topology before it. Neither was defined, so on a host with no
+ # /dev/kfd the script fell through to `echo FIRED` and looked like it worked,
+ # and on a host WITH one it exited 127 from an undefined substep.
+ 'substep() { echo "$1"; }',
+ 'C_WARN=""',
+ # False, because the arm under test is the third of three and the first
+ # requires a topology. Stubbed rather than read: on a real gfx1151 the
+ # topology names an AMD GPU and the first arm would answer instead.
+ "_kfd_topology_has_an_amd_gpu() { return 1; }",
+ guard,
+ " echo FIRED",
+ "fi",
+ ]
+ )
+ # The arm under test requires the node to be ABSENT. Owned by the case, so a runner
+ # that has one does not answer for it.
+ script = _kfd_node_the_case_owns(script, tmp_path, present = False)
+ # The set arrives as an exported variable rather than a generated assignment: a repr()
+ # inside shell single quotes turns the newline separating two nodes into a literal
+ # backslash-n, which reads as one unmatched line and looks like the suppression failing.
+ out = subprocess.run(
+ ["bash", "-c", script],
+ capture_output = True,
+ text = True,
+ check = True,
+ env = {**os.environ, "_closed_amd_nodes": closed_nodes},
+ )
+ return "FIRED" in out.stdout
+
+
+ # /dev/kfd existing is the evidence the stack is already loaded, so telling the user
+ # to install one cannot help; the group advice after the case is the repair.
+ pytest.param(("/dev/kfd", True, False), id = "a_closed_kfd_node_suppresses_it"),
+ pytest.param(("/dev/kfd\n/dev/dri/renderD128", True, False),
+ id = "a_closed_pair_suppresses_it"),
+ # The case the suppression must not swallow: no /dev/kfd at all, and a render node
+ # this account cannot open. No membership creates /dev/kfd, so both have to print.
+ pytest.param(("/dev/dri/renderD128", True, True), id = "a_missing_kfd_node_keeps_it"),
+ # The negative control: the branch's original behaviour is untouched.
+ pytest.param(("", True, True), id = "nothing_closed_keeps_it"),
+ # And that the route variable is consulted rather than merely computed.
+ pytest.param(("", False, False), id = "the_route_gate_actually_suppresses_it"),
+])
+def test_when_the_installers_kernel_stack_hint_fires(tmp_path, case):
+ """Which hosts install.sh tells to install a ROCm kernel stack."""
+ closed_nodes, route, fires = case
+ assert _kernel_stack_hint_runs(tmp_path, closed_nodes, route = route) is fires
+
+
+def _stat_nodes(monkeypatch, modes: dict, names: dict):
+ """Stub os.stat and grp for the node set ``modes`` maps to (gid, mode).
+
+ Paths outside the map raise, which covers the vanished-node arm. The stub takes
+ ``follow_symlinks`` because pytest itself stats files while this patch is in force, and a
+ two-argument lambda takes the whole session down rather than failing the test.
+ """
+
+ def _stat(path, *, follow_symlinks = True):
+ if str(path) not in modes:
+ raise OSError("gone")
+ _entry = modes[str(path)]
+ gid, mode = _entry[0], _entry[1]
+ # Real device nodes are root-owned and POSIX consults the owner class first, so a
+ # fake without st_uid would take the owner branch on whatever uid the runner has --
+ # often root on CI, which would assert nothing about the group classification these
+ # cases name. The tests that DO cover owner precedence build real files in tmp_path.
+ uid = _entry[2] if len(_entry) > 2 else 0
+ if uid == amd.os.getuid():
+ uid += 1
+ return type("st", (), {"st_gid": gid, "st_mode": mode, "st_uid": uid})()
+
+ def _getgrgid(gid):
+ if gid not in names:
+ raise KeyError(gid)
+ return type("gr", (), {"gr_name": names[gid]})()
+
+ monkeypatch.setattr(amd.os, "stat", _stat)
+ monkeypatch.setattr(grp, "getgrgid", _getgrgid)
+ # Synthetic nodes have no ACL and the account starts outside their groups. Membership
+ # tests override these defaults explicitly after building the nodes.
+ monkeypatch.setattr(amd, "_has_an_access_acl", lambda _path: False)
+ monkeypatch.setattr(amd.os, "getgid", lambda: 1)
+ monkeypatch.setattr(amd.os, "getgroups", lambda: [])
+
+
+_THREE_NODES = {
+ "/dev/kfd": (44, 0o660),
+ "/dev/dri/renderD128": (44, 0o660),
+ "/dev/dri/renderD129": (39, 0o660),
+}
+
+
+ # Two nodes owned by one group name it once, and order is first-seen so the command
+ # reads like the node list.
+ pytest.param((_THREE_NODES, {44: "video", 39: "render"},
+ ["/dev/dri/renderD129", "/dev/kfd", "/dev/dri/renderD128"],
+ dict(joinable = ["render", "video"])),
+ id = "the_group_derivation_reads_the_node"),
+ # The container case docker/run.sh documents: --group-add passes the host's numeric
+ # gids and no group entry inside matches them. Fails before the fix, which returned
+ # the bare number for usermod to consume.
+ pytest.param(({"/dev/kfd": (993, 0o660)}, {}, ["/dev/kfd"], dict(unnamed = [993])),
+ id = "a_gid_with_no_group_entry_is_reported_rather_than_prescribed"),
+ # A udev rule leaving a node root:render 0600 denies the group as well, so joining
+ # render opens nothing. Read the mode before naming the group. Fails before the fix,
+ # which read st_gid alone and would have prescribed render.
+ pytest.param(({"/dev/kfd": (44, 0o600)}, {44: "render"}, ["/dev/kfd"],
+ dict(no_group = ["/dev/kfd"])),
+ id = "a_node_whose_own_group_cannot_open_it_is_not_a_membership_problem"),
+ # Its boundary: the probe's own bar is read-write, so 0640 is not a joinable group.
+ pytest.param(({"/dev/kfd": (44, 0o640)}, {44: "render"}, ["/dev/kfd"],
+ dict(no_group = ["/dev/kfd"])),
+ id = "group_read_without_write_is_not_enough"),
+ # And the failure mode that must not raise: diagnostics run where things are already
+ # wrong, so a node that vanished between the probe and the message drops out.
+ pytest.param(({"/dev/dri/renderD128": (44, 0o660)}, {44: "video"},
+ ["/dev/kfd", "/dev/dri/renderD128"], dict(joinable = ["video"])),
+ id = "a_node_that_cannot_be_stat_contributes_nothing"),
+ # root:root 0660 opens for anyone in the root group, so the derivation would accept
+ # the name and hand it to usermod. That grants a great deal besides the GPU.
+ pytest.param(({"/dev/kfd": (0, 0o660, 0)}, {0: "root"}, ["/dev/kfd"],
+ dict(privileged = ["root"])),
+ id = "a_root_owned_node_is_not_answered_with_usermod_root"),
+ # The control: render is not privileged, so the same shape still yields the command.
+ pytest.param(({"/dev/kfd": (39, 0o660, 0)}, {39: "render"}, ["/dev/kfd"],
+ dict(joinable = ["render"])),
+ id = "an_ordinary_owning_group_is_still_prescribed"),
+])
+def test_the_group_derivation_over_a_node_set(monkeypatch, case):
+ """The helper itself, since every hint case above stubs it. Every bucket a case does not
+ name has to come back empty, or the hint offers a repair the node does not support."""
+ modes, names, paths, buckets = case
+ _stat_nodes(monkeypatch, modes, names)
+ assert amd._groups_that_own(paths) == _buckets(**buckets)
+
+
+# The sentence the kernel-stack branch prints. The harnesses that lift that branch anchor on
+# it rather than on either condition: the mapping one is what an earlier change edited, and a
+# revert that stopped the extraction finding anything would read "the text changed" as "the
+# behaviour changed". _amd_gpu_present_via_pci is named twice, so it is not an anchor either.
+_PCI_SENTENCE = "An AMD GPU is on the PCI bus but ROCm cannot see it"
+
+
+def _install_sh_lines() -> "list[str]":
+ """install.sh, split into lines. Every harness below lifts what it needs out of this."""
+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"
+ return install_sh.read_text(encoding = "utf-8").splitlines()
+
+
+# The one thing a lifted guard reads that no stub can reach: `[ -e /dev/kfd ]`. A test
+# operator is not a command, so it cannot be shadowed by a function, and on a host that
+# really has an AMD GPU the node really is there -- so the branch taken was decided by the
+# runner rather than by the case. That is why this file passed on a box with no AMD card
+# and failed on the gfx1151 the feature exists for.
+#
+# Only the PATH is redirected, and only inside the existence tests. The condition, its
+# ordering and its `&&` chain are still lifted verbatim, exactly as _has_amd_rocm_gpu and
+# _amd_gpu_present_via_pci are stubbed rather than restated. The node name inside the
+# printed sentences is untouched, because the arms below assert on that text.
+_KFD_EXISTENCE_TEST = re.compile(r"(\[ +!? ?-e +)/dev/kfd\b")
+
+
+def _kfd_node_the_case_owns(script: str, tmp_path, *, present: bool) -> str:
+ """Point install.sh's `-e /dev/kfd` tests at a node this case creates or withholds."""
+ node = tmp_path / "kfd"
+ if present:
+ node.write_bytes(b"")
+ elif node.exists():
+ node.unlink()
+ redirected, n = _KFD_EXISTENCE_TEST.subn(rf"\g<1>{node}", script)
+ # A guard that stopped containing the test would silently go back to reading the host,
+ # and every arm here would agree for the wrong reason.
+ assert n, "install.sh no longer tests `-e /dev/kfd` where this harness expects it"
+ return redirected
+
+
+def _install_sh_if_above(lines: "list[str]", i: int) -> int:
+ """Walk back from a line inside a condition to the `if` opening its block.
+
+ A multi-line condition puts the `if` and its last test on different lines, so requiring
+ both on one line stopped finding anything the moment a gate was added -- and a harness
+ that finds nothing raises here rather than silently testing a shorter script.
+ """
+ while not lines[i].lstrip().startswith("if "):
+ i -= 1
+ return i
+
+
+def _install_sh_anchor(lines: "list[str]", text: str) -> int:
+ """The index of the line containing ``text``."""
+ return next(i for i, line in enumerate(lines) if text in line)
+
+
+def _install_sh_if(lines: "list[str]", tail: str) -> int:
+ """The index of the `if` opening the block whose condition ENDS with ``tail``."""
+ return _install_sh_if_above(
+ lines, next(j for j, line in enumerate(lines) if line.rstrip().endswith(tail))
+ )
+
+
+def _shell_fn(lines: "list[str]", name: str) -> str:
+ """One function definition lifted out of install.sh, by brace depth.
+
+ Slicing from a definition down to the block under test worked only while the two were
+ adjacent, and a shell function must be defined before the line that calls it, so the
+ run-scope predicates now sit above the diagnoses they gate and each is lifted by name. A
+ miss raises rather than returning "": an undefined function would fail the block under
+ test for a reason unrelated to the case.
+ """
+ start = next(i for i, line in enumerate(lines) if line.startswith(f"{name}() {{"))
+ depth = 0
+ for end in range(start, len(lines)):
+ depth += lines[end].count("{") - lines[end].count("}")
+ if depth == 0:
+ return "\n".join(lines[start : end + 1])
+ raise AssertionError(f"unterminated {name}() in install.sh")
+
+
+def _run_scope_defs(lines: "list[str]", *, nvidia: bool = False) -> "list[str]":
+ """The run-scope predicates the install.sh harnesses below share.
+
+ Lifted rather than restated so every arm goes through the installer's own rule.
+ _is_pip_rocm_family_leaf comes with _torch_opens_amd_nodes, which classifies a ROCm index
+ through it; _shell_quote with anything that pastes a command, since without it every
+ interpolated name comes back EMPTY and the arm reads as a command naming nobody; and
+ _requested_llama_backend because all three predicates read the request through it, which
+ is what makes the legacy UNSLOTH_FORCE_VULKAN reach them as in a real run.
+ _has_usable_nvidia_gpu is the exception, stubbed because the real one runs nvidia-smi and
+ would answer from the runner's hardware -- false by default, so each arm is AMD-only.
+ """
+ return [
+ _shell_fn(lines, "_requested_llama_backend"),
+ _shell_fn(lines, "_torch_index_url_leaf"),
+ _shell_fn(lines, "_is_pip_rocm_family_leaf"),
+ _shell_fn(lines, "_torch_opens_amd_nodes"),
+ f"_has_usable_nvidia_gpu() {{ return {0 if nvidia else 1}; }}",
+ _shell_fn(lines, "_auto_bundle_opens_amd_nodes"),
+ _shell_fn(lines, "_run_may_open_kfd"),
+ _shell_fn(lines, "_shell_quote"),
+ ]
+
+
+def _install_sh_run(script: str, *, env: "dict | None" = None) -> str:
+ """Run a lifted script under bash and return its stdout, failing with its own stderr."""
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, env = env)
+ assert out.returncode == 0, out.stderr
+ return out.stdout
+
+
+def _install_sh_env(
+ closed_nodes: str,
+ env_user: str,
+ backend: "str | None",
+ torch_index: str = "https://download.pytorch.org/whl/rocm6.4",
+) -> dict:
+ """The environment install.sh reads: the closed set, the account, the request, and the
+ torch index, which the run-scope predicates read to tell a CPU wheel from a ROCm one.
+ Defaulted to a ROCm index so every existing case keeps the run it was written for."""
+ env = {
+ **os.environ,
+ "_closed_amd_nodes": closed_nodes,
+ "USER": env_user,
+ "TORCH_INDEX_URL": torch_index,
+ }
+ env.pop("UNSLOTH_LLAMA_CPP_BACKEND", None)
+ if backend is not None:
+ env["UNSLOTH_LLAMA_CPP_BACKEND"] = backend
+ return env
+
+
+def _the_real_stat_derivation_runs_here() -> None:
+ """Skip a case that lifts install.sh's REAL stat|awk classifier onto this host.
+
+ install.sh reads each node with GNU ``stat -c '%a|%g|%u|%n|%G'``. BSD stat on macOS
+ rejects -c outright, so the record arrives empty, the owner branch cannot match a uid it
+ never parsed, and every node falls through to "join a group" -- an artifact of running
+ the harness there, not a finding about the installer, which gates every diagnosis that
+ reaches this classifier on `[ "$OS" != "macos" ]` and so never runs it on a Mac at all.
+
+ Keyed on the OS, and never on hardware: every Linux host runs all of these whatever GPU
+ it has, which is the property the rest of this file was fixed to keep. The synthetic-record
+ harness beside this one stubs stat and therefore still runs everywhere.
+ """
+ if platform.system() != "Linux":
+ pytest.skip(
+ "install.sh's stat|awk node classifier needs GNU stat -c, and the installer "
+ "gates every diagnosis that uses it on OS != macos"
+ )
+
+
+def _install_sh_hint(
+ closed_nodes: str,
+ *,
+ render_present: bool = True,
+ amd_present: bool = True,
+ self_uid: str = "4242",
+ self_gids: str = "65534",
+ render_open: bool = False,
+ repairs: "str | None" = None,
+ skip_torch: bool = False,
+ backend: "str | None" = None,
+ torch_index: str = "https://download.pytorch.org/whl/rocm6.4",
+ env_user: str = "ada",
+ id_user: "str | None" = "ada",
+ nvidia: bool = False,
+) -> str:
+ """The installer's closed-node message, run for a given closed set.
+
+ Lifted from install.sh rather than restated, and the whole block rather than a condition,
+ because the thing under test is the sentence it prints. substep is stubbed to plain echo;
+ the node list comes in through the environment, since embedding it in the script would
+ put a literal backslash-n inside shell quotes and turn two nodes into one unmatched line.
+ """
+ # repairs None means the REAL derivation runs, reading this host's own stat(1). A
+ # stubbed one is a synthetic record and stays portable, so only the real arm is gated.
+ if repairs is None:
+ _the_real_stat_derivation_runs_here()
+ lines = _install_sh_lines()
+ start = _install_sh_if(lines, '[ -n "$_closed_amd_nodes" ]; then')
+ end = next(i for i in range(start, len(lines)) if lines[i] == "fi")
+ block = "\n".join(lines[start : end + 1])
+
+ helper = _shell_fn(lines, "_amd_node_repairs")
+
+ script = "\n".join(
+ [
+ 'substep() { echo "$1"; }',
+ 'C_WARN=""',
+ # Stubbed rather than lifted, along with _an_amd_render_node_is_open and
+ # _has_usable_nvidia_gpu below: the real ones read /sys and /dev or run
+ # nvidia-smi, so a live one would answer from the runner's own hardware. All
+ # three are false by default, so an arm reads as the AMD-only host it describes.
+ f"_amd_render_node_present() {{ return {0 if render_present else 1}; }}",
+ # A real device node is root-owned; a tmp_path node belongs to the runner, and
+ # the installer stops at the owner class when those match, so the arms choose
+ # which case they test. Both spellings: the owner-class test asks `id -u` and the
+ # repair asks `id -un`, and a stub answering one for the other names a uid as an
+ # account. id_user None is a uid with no passwd entry, where the real `id -un`
+ # FAILS: the shape of `docker run --user 1234`, and why the container repair
+ # exists.
+ (
+ # printf with the value single-quoted, not echo: a name carrying a backslash
+ # is de-escaped by the stub itself otherwise, and the arm testing how such a
+ # name is QUOTED then never sees one.
+ f'id() {{ case "$1" in -un) printf %s\\\\n {shlex.quote(id_user or "")} ;; '
+ f'-G) echo "{self_gids}" ;; *) echo {self_uid} ;; esac; }}'
+ if id_user is not None
+ else f'id() {{ case "$1" in -un) return 1 ;; '
+ f'-G) echo "{self_gids}" ;; *) echo {self_uid} ;; esac; }}'
+ ),
+ f"_kfd_topology_has_an_amd_gpu() {{ return {0 if amd_present else 1}; }}",
+ f"_an_amd_render_node_is_open() {{ return {0 if render_open else 1}; }}",
+ # The route the diagnoses are gated on; the gate has its own tests below.
+ "_amd_node_diag_route=true",
+ "OS=linux",
+ # The run-scope predicate the block asks. Lifted rather than stubbed, so the
+ # default arms go through the same rule the installer applies.
+ f"SKIP_TORCH={'true' if skip_torch else 'false'}",
+ f"_has_usable_nvidia_gpu() {{ return {0 if nvidia else 1}; }}",
+ _shell_fn(lines, "_auto_bundle_opens_amd_nodes"),
+ _shell_fn(lines, "_run_may_open_a_gpu_node"),
+ # The block also asks which nodes THIS run opens, to name the right --device
+ # pair, so the predicate has to exist before the span that calls it.
+ _shell_fn(lines, "_torch_index_url_leaf"),
+ # _torch_opens_amd_nodes classifies a ROCm index through _is_pip_rocm_family_leaf
+ # and every scope predicate reads the request through _requested_llama_backend,
+ # so a harness that omits either measures a missing function rather than a rule.
+ _shell_fn(lines, "_is_pip_rocm_family_leaf"),
+ _shell_fn(lines, "_requested_llama_backend"),
+ _shell_fn(lines, "_torch_opens_amd_nodes"),
+ _shell_fn(lines, "_run_may_open_kfd"),
+ # The block quotes every name it pastes into a command through this. Lifted, not
+ # stubbed: without it the substitutions come back EMPTY and the arms below read
+ # as commands that name nobody.
+ _shell_fn(lines, "_shell_quote"),
+ # The real derivation by default. An override stands in only where the case
+ # cannot be built on disk -- a node whose GID has no entry in the group
+ # database -- and _amd_node_repairs has its own tests either way.
+ helper if repairs is None else f"_amd_node_repairs() {{ printf '%s\\n' '{repairs}'; }}",
+ block,
+ ]
+ )
+ out = subprocess.run(
+ ["bash", "-c", script],
+ capture_output = True,
+ text = True,
+ env = _install_sh_env(closed_nodes, env_user, backend, torch_index),
+ )
+ assert out.returncode == 0, out.stderr
+ return out.stdout
+
+
+def _a_node_a_membership_would_open(tmp_path, *, mode: int = 0o660):
+ """A fixture node whose owning group is one the rule calls JOINABLE, and its name.
+
+ A tmp_path file gets the runner's primary group, and on a root CI runner that group is
+ root -- which both halves classify as privileged and refuse to prescribe, so the arms
+ below would assert the privileged sentence instead of the derivation. The group is
+ therefore CHOSEN rather than inherited: root may chgrp to any group, an ordinary account
+ only to one it belongs to, and it skips when all of those are privileged.
+ """
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(mode)
+ _candidates = (
+ [_g.gr_gid for _g in grp.getgrall()]
+ if os.geteuid() == 0
+ else [os.getgid(), *os.getgroups()]
+ )
+ for _gid in _candidates:
+ try:
+ _name = grp.getgrgid(_gid).gr_name
+ except KeyError:
+ continue
+ if _gid == 0 or _name in amd._PRIVILEGED_GROUPS:
+ continue
+ try:
+ os.chown(node, -1, _gid)
+ except OSError:
+ continue
+ return node, _name
+ pytest.skip("every group this account can use is one the rule refuses to prescribe")
+
+
+def test_the_group_fixture_never_hands_back_a_group_the_rule_refuses(tmp_path, monkeypatch):
+ """The guard for it, since a fixture that quietly picks a privileged group does not fail,
+ it makes the arms below assert the wrong sentence -- which is how this was found. Stands
+ in for the root runner by making the group this account would otherwise inherit
+ privileged, so the search has to move off it."""
+ monkeypatch.setattr(
+ amd,
+ "_PRIVILEGED_GROUPS",
+ frozenset(amd._PRIVILEGED_GROUPS | {grp.getgrgid(os.getgid()).gr_name}),
+ )
+ _node, _group = _a_node_a_membership_would_open(tmp_path)
+ assert _group not in amd._PRIVILEGED_GROUPS
+
+
+def test_the_installer_names_the_group_the_node_actually_has(tmp_path):
+ """The shell half of the same item, and its only arm that reads a real file: the message
+ must name the group that owns the node it refused. Fails before the fix, which printed
+ render,video for every host. The group is read with the same stat the installer uses,
+ since a runner's primary group is not knowable in advance -- but asserting it is NOT
+ render,video is what makes that comparison mean something. 0660 is the mode a real render
+ node has, and a default 0644 is a node no membership opens."""
+ node, _group = _a_node_a_membership_would_open(tmp_path)
+ owner = subprocess.run(
+ ["stat", "-c", "%G", str(node)],
+ capture_output = True,
+ text = True,
+ ).stdout.strip()
+ out = _install_sh_hint(str(node))
+ assert f"usermod -a -G {owner} ada" in out
+ if owner not in ("render", "video"):
+ assert "render,video" not in out
+
+
+def test_the_installer_falls_back_when_the_node_is_gone(tmp_path):
+ """The control: a path that cannot be stat'd still has to produce advice, and an empty -G
+ argument would be worse than the hard-coded pair it replaced."""
+ out = _install_sh_hint(str(tmp_path / "renderD128"))
+ assert "usermod -a -G render,video ada" in out
+
+
+def _reason_with_mask(monkeypatch, var: str, value: str, backends: set) -> str:
+ """The empty-probe reason on a closed-node host carrying one visibility mask."""
+ _nodes(monkeypatch, present = _AMD_NODES, openable = set())
+ for other in _GPU_MASK_VARS:
+ monkeypatch.delenv(other, raising = False)
+ monkeypatch.setenv(var, value)
+ _ggml(monkeypatch, backends)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_is_vulkan_backend",
+ staticmethod(lambda _b: backends == {"vulkan"}),
+ )
+ return _empty_probe()
+
+
+ # HIP_VISIBLE_DEVICES=0 names a device rather than hiding one, so it is not why the
+ # probe came back empty and reporting it sends the user after a fix that cannot help,
+ # on top of the one that can. Fails before the fix, which listed every variable that
+ # was merely SET.
+ pytest.param(("HIP_VISIBLE_DEVICES", "0", {"hip"}, (_USERMOD,), ("visibility mask",)),
+ id = "a_selector_that_still_exposes_a_device_is_not_a_second_blocker"),
+ # The control: -1 names no device, so that host really does need both fixes. Not an
+ # EMPTY value, which is the one thing clr's parser is never entered on: the guard is
+ # on the first byte, so an empty HIP mask is not a filter and is not the variable clr
+ # reads either.
+ pytest.param(("HIP_VISIBLE_DEVICES", "-1", {"hip"}, (_USERMOD, "HIP_VISIBLE_DEVICES='-1'"), ()),
+ id = "a_mask_that_hides_everything_is_still_reported"),
+ # CUDA and HIP parse the list left to right and stop at the first entry that names no
+ # device, so -1 leading the list leaves nothing enumerated.
+ pytest.param(("CUDA_VISIBLE_DEVICES", "-1", {"hip"},
+ ("CUDA_VISIBLE_DEVICES='-1'", "visibility mask is also in force"), ()),
+ id = "a_negative_first_entry_hides_everything"),
+ # And its control: 0,-1 stops at the -1 but has already exposed GPU 0. Without this
+ # the fix could be "any minus sign anywhere hides everything", which passes the case
+ # above and is wrong.
+ pytest.param(("CUDA_VISIBLE_DEVICES", "0,-1", {"hip"}, (), ("visibility mask",)),
+ id = "a_leading_valid_entry_survives_a_later_invalid_one"),
+ # A Vulkan-only install reads none of these four, so an inherited HIP or CUDA mask is
+ # not a blocker for it at any value -- the render node it cannot open is.
+ pytest.param(("HIP_VISIBLE_DEVICES", "-1", {"vulkan"}, (_USERMOD,), ("visibility mask",)),
+ id = "a_vulkan_build_is_not_told_about_a_mask_it_never_reads"),
+ # Its control, one backend apart: the identical environment must still report the
+ # mask when the install is one that actually reads it.
+ pytest.param(("HIP_VISIBLE_DEVICES", "-1", {"hip"}, ("visibility mask is also in force",), ()),
+ id = "the_same_hiding_mask_still_counts_for_a_hip_build"),
+])
+def test_which_lone_masks_are_reported(monkeypatch, linux, case):
+ """Whether one inherited visibility mask is named as a second blocker beside the groups.
+ Every arm runs on the same host, with every AMD node closed."""
+ var, value, backends, says, does_not_say = case
+ reason = _reason_with_mask(monkeypatch, var, value, backends)
+ _asserts(reason, says, does_not_say)
+
+
+def _installer_index_summary(
+ index_url: str,
+ closed_nodes: str,
+ tmp_path,
+ *,
+ nvidia: bool = False,
+) -> str:
+ """install.sh's index summary and the two diagnoses that follow it, run for one index.
+
+ The whole span is lifted rather than the guard alone, because the thing under test is
+ WHERE the diagnosis sits relative to the case: a copy of the condition would answer the
+ same whichever arm it had been left in.
+
+ The three arms of that case are told apart by ``-e /dev/kfd`` and the KFD topology, and
+ both of those are the HOST's until this harness takes them: the case here is a machine
+ with no runtime, so the node is withheld and the topology stubbed empty. Left to the
+ host these cases passed on a box with no AMD GPU and failed on the hardware the feature
+ is for, which is the wrong way round for every one of them.
+ """
+ lines = _install_sh_lines()
+ start = max(i for i, line in enumerate(lines) if line == 'case "$TORCH_INDEX_URL" in')
+ anchor = next(i for i in range(start, len(lines)) if "needs a recent kernel" in lines[i])
+ # Through the closed-node block as well, so one run shows which diagnosis this index gets.
+ last = next(i for i in range(anchor, len(lines)) if "membership opens it" in lines[i])
+ end = next(i for i in range(last, len(lines)) if lines[i] == "fi")
+ script = "\n".join(
+ [
+ _shell_fn(lines, "_amd_node_repairs"),
+ 'substep() { echo "$1"; }',
+ 'C_WARN=""',
+ "_amd_gpu_radeon=false",
+ '_strip_index_url_credentials() { printf "%s\\n" "$1"; }',
+ "_has_amd_rocm_gpu() { return 1; }", # ROCm cannot see the card
+ "_amd_gpu_present_via_pci() { return 0; }", # but the PCI bus can
+ "SKIP_TORCH=false",
+ "OS=linux",
+ "_amd_render_node_present() { return 0; }",
+ "_kfd_topology_has_an_amd_gpu() { return 1; }",
+ # The route gate classifies the index by its canonical leaf, so the classifiers
+ # are lifted rather than stubbed, or the per-URL cases below would assert about
+ # the stub. They are defined above the case in install.sh, so the span lifted
+ # below calls them without carrying them.
+ *_run_scope_defs(lines, nvidia = nvidia),
+ _shell_fn(lines, "_run_may_open_a_gpu_node"),
+ *lines[start : end + 1],
+ ]
+ )
+ script = _kfd_node_the_case_owns(script, tmp_path, present = False)
+ out = subprocess.run(
+ ["bash", "-c", script],
+ capture_output = True,
+ text = True,
+ env = {
+ **os.environ,
+ "TORCH_INDEX_URL": index_url,
+ "_closed_amd_nodes": closed_nodes,
+ },
+ )
+ assert out.returncode == 0, out.stderr
+ return out.stdout
+
+
+_GFX_INDEX = "https://repo.radeon.com/rocm/manylinux/gfx1151"
+
+
+ # The runtime-less host this was written for: no /dev/kfd, so the per-arch reroute
+ # rewrote its cpu index to a gfx one and it took the other arm of the case. Left
+ # inside the */cpu arm the diagnosis never printed for the host that needed it. Fails
+ # before the fix, which produced only the wheels line for this index.
+ pytest.param((_GFX_INDEX, "", ("ROCm cannot see it",), ()),
+ id = "the_kernel_stack_diagnosis_reaches_a_rerouted_gfx_index"),
+ # The control: the arm the diagnosis used to live in must keep it, or the hoist has
+ # moved the message rather than widened it.
+ pytest.param(("https://download.pytorch.org/whl/cpu", "", ("ROCm cannot see it",), ()),
+ id = "a_cpu_index_on_the_same_host_still_gets_it"),
+ # And the suppression the hoist must carry with it: on a gfx index too, an existing
+ # /dev/kfd means the group advice is the repair and the kernel stack is not.
+ pytest.param((_GFX_INDEX, "/dev/kfd", ("cannot open its device nodes",),
+ ("ROCm cannot see it",)),
+ id = "a_closed_kfd_node_still_suppresses_it_after_the_case"),
+])
+def test_which_index_summary_carries_the_kernel_stack_diagnosis(tmp_path, case):
+ """Which wheel index the diagnosis prints for, now that it sits after the case."""
+ index_url, closed_nodes, says, does_not_say = case
+ out = _installer_index_summary(index_url, closed_nodes, tmp_path)
+ _asserts(out, says, does_not_say)
+
+
+def _reason_with_masks(
+ monkeypatch,
+ env: dict,
+ backends: set,
+ gpu_count: "int | None" = None,
+) -> str:
+ """The empty-probe reason on a closed-node host carrying several visibility masks.
+
+ ``gpu_count`` is what KFD enumerates, so a selector can be judged against something. None
+ is the default because it is what an unreadable topology answers, which is the state
+ every test written before that check ran in.
+ """
+ _nodes(monkeypatch, present = _AMD_NODES, openable = set())
+ monkeypatch.setattr(amd, "amd_kfd_gpu_node_count", lambda: gpu_count)
+ for var in (
+ "CUDA_VISIBLE_DEVICES",
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "GPU_DEVICE_ORDINAL",
+ "GGML_VK_VISIBLE_DEVICES",
+ ):
+ monkeypatch.delenv(var, raising = False)
+ for var, value in env.items():
+ monkeypatch.setenv(var, value)
+ _ggml(monkeypatch, backends)
+ monkeypatch.setattr(
+ LlamaCppBackend,
+ "_is_vulkan_backend",
+ staticmethod(lambda _b: backends == {"vulkan"}),
+ )
+ return _empty_probe()
+
+
+_UUID = "GPU-4b2c9f1e0a7d3b58"
+_IN_FORCE = "visibility mask is also in force"
+
+
+ # _gpu_device_ordinal_active reads a whitespace GPU_DEVICE_ORDINAL as no filter, so
+ # an empty one hides nothing. Fails before the fix, which applied the CUDA/HIP
+ # first-token rule to all four names alike and named a variable already inert.
+ pytest.param(({"GPU_DEVICE_ORDINAL": ""}, None, (_USERMOD,), ("visibility mask",)),
+ id = "an_empty_ordinal_variable_is_not_a_filter"),
+ # Its control: a value that IS a filter and whose first entry names no device leaves
+ # nothing enumerated, so that host needs both fixes.
+ pytest.param(({"GPU_DEVICE_ORDINAL": "-1"}, None, (_IN_FORCE, "GPU_DEVICE_ORDINAL='-1'"), ()),
+ id = "an_ordinal_that_hides_everything_is_still_reported"),
+ # clr reads HIP_VISIBLE_DEVICES when it is non-empty and CUDA_VISIBLE_DEVICES
+ # otherwise, so an empty CUDA mask underneath a valid HIP one is never looked at.
+ # Fails before the fix, which judged each of the four on its own value.
+ pytest.param(({"HIP_VISIBLE_DEVICES": "0", "CUDA_VISIBLE_DEVICES": ""}, None, (_USERMOD,),
+ ("visibility mask",)),
+ id = "an_empty_cuda_mask_behind_a_valid_hip_one_is_not_consulted"),
+ # The control, one variable apart: with no HIP mask above it the same empty CUDA
+ # value is the one clr reads, and it exposes nothing.
+ pytest.param(({"CUDA_VISIBLE_DEVICES": ""}, None, ("CUDA_VISIBLE_DEVICES is empty", _IN_FORCE),
+ ()),
+ id = "the_same_empty_cuda_mask_blocks_once_hip_is_unset"),
+ # ROCr is a LOWER layer than clr and composes rather than defers: it filters the
+ # agent list hsa_iterate_agents returns and the HIP ordinals index what it left, so
+ # an empty ROCr mask leaves nothing to index whatever HIP says. This is what keeps
+ # the fix from being "only the winner of the precedence chain counts".
+ pytest.param(({"HIP_VISIBLE_DEVICES": "0", "ROCR_VISIBLE_DEVICES": ""}, None,
+ ("ROCR_VISIBLE_DEVICES is empty", _IN_FORCE), ()),
+ id = "an_empty_rocr_mask_blinds_the_runtime_under_a_valid_hip_one"),
+ # HIP stops at the first index no device answers to, so HIP_VISIBLE_DEVICES=3 on a
+ # one-GPU host exposes nothing -- which is the empty probe being explained, and was
+ # read as a valid selector.
+ pytest.param(({"HIP_VISIBLE_DEVICES": "3"}, 1, (_IN_FORCE, "HIP_VISIBLE_DEVICES='3'"), ()),
+ id = "an_ordinal_naming_a_device_that_is_not_there_hides_everything"),
+ # The control: the same host and variable pointing at a GPU it has. Without it the
+ # fix could be "any ordinal blocks", which would take the GPU off every host with a
+ # legitimate selector.
+ pytest.param(({"HIP_VISIBLE_DEVICES": "0"}, 1, (), (_IN_FORCE,)),
+ id = "an_ordinal_that_does_name_a_device_is_still_not_a_blocker"),
+ # The other control: an unreadable KFD topology is a detection miss, and reading it
+ # as "no devices" would call every selector on the host a blocker.
+ pytest.param(({"HIP_VISIBLE_DEVICES": "3"}, None, (), (_IN_FORCE,)),
+ id = "an_unreadable_device_count_leaves_the_selector_alone"),
+ # ROCr accepts a UUID as well as an ordinal, and one naming no device stops the list
+ # exactly as a bad ordinal does. Nothing here can match a UUID against the KFD count,
+ # so it is reported as unresolved rather than dismissed, which is what the ordinal
+ # check did. Reported, not judged: claiming it blocks would invent a fault this
+ # cannot see. Fails before the fix, which said nothing for any non-digit entry.
+ pytest.param(({"ROCR_VISIBLE_DEVICES": _UUID}, 1, ("cannot resolve", "ROCR_VISIBLE_DEVICES"),
+ ("which the groups do not clear",)),
+ id = "a_rocr_selector_naming_a_uuid_is_reported_as_unresolved"),
+ # The control that keeps it narrow: an ordinal the count can resolve is judged as
+ # before, so the new sentence cannot appear on every host that sets the variable.
+ pytest.param(({"ROCR_VISIBLE_DEVICES": "0"}, 2, (), ("cannot resolve", "visibility mask")),
+ id = "a_rocr_ordinal_that_names_a_device_is_still_left_alone"),
+ # The other boundary. An earlier revision asserted the opposite here, on the claim
+ # that only ROCr accepts a UUID; rocdevice.cpp refutes it, matching a "GPU-" token
+ # against each agent's own HSA_AMD_AGENT_INFO_UUID before falling back to an ordinal.
+ # So the token may name a device or nothing, as under ROCr, and nothing here can tell
+ # which. The count is deliberately known here: a UUID is not an index into it.
+ pytest.param(({"HIP_VISIBLE_DEVICES": _UUID}, 1, ("names a device this cannot resolve",),
+ (_IN_FORCE,)),
+ id = "a_uuid_in_the_hip_layer_is_unresolved_rather_than_a_blocker"),
+ # ROCr filters the physical list first and renumbers the survivors; HIP indexes
+ # those. With ROCR_VISIBLE_DEVICES=0 on a two-GPU host one survives, so HIP ordinal 1
+ # names nothing, while against the physical count of 2 it reads as a valid selector.
+ # Fails before the fix, which used the KFD count for every layer.
+ pytest.param(({"ROCR_VISIBLE_DEVICES": "0", "HIP_VISIBLE_DEVICES": "1"}, 2,
+ ("HIP_VISIBLE_DEVICES='1'", "which the groups do not clear"), ()),
+ id = "a_hip_ordinal_is_judged_against_what_rocr_left"),
+ # The control: ROCr leaving both devices makes HIP ordinal 1 a real device again, so
+ # the composed reading must not call every stacked pair a blocker.
+ pytest.param(({"ROCR_VISIBLE_DEVICES": "0,1", "HIP_VISIBLE_DEVICES": "1"}, 2, (),
+ ("which the groups do not clear",)),
+ id = "the_same_ordinal_inside_what_rocr_left_is_not_a_blocker"),
+ # And the boundary: a UUID in the ROCr layer means the survivors cannot be counted, so
+ # the HIP ordinal is judged against nothing rather than against an invented count.
+ pytest.param(({"ROCR_VISIBLE_DEVICES": _UUID, "HIP_VISIBLE_DEVICES": "1"}, 2, (),
+ ("which the groups do not clear",)),
+ id = "an_unresolvable_rocr_entry_leaves_the_hip_ordinal_alone"),
+])
+def test_which_stacked_masks_are_reported(monkeypatch, linux, case):
+ """How the four selectors compose, on a HIP build with every AMD node closed. The groups
+ never clear a mask, so anything named here is a second repair the user also needs."""
+ env, gpu_count, says, does_not_say = case
+ reason = _reason_with_masks(monkeypatch, env, {"hip"}, gpu_count)
+ _asserts(reason, says, does_not_say)
+
+
+def test_the_installer_says_the_same_thing_about_a_missing_render_node(tmp_path):
+ """The shell half of the container case. Stubbed on both sides so the arms differ only in
+ the answer, since the real helper reads the runner's own /sys and /dev."""
+ node = tmp_path / "kfd"
+ node.write_bytes(b"")
+ node.chmod(0o660)
+ assert "No AMD render node" in _install_sh_hint(str(node), render_present = False)
+ assert "No AMD render node" not in _install_sh_hint(str(node), render_present = True)
+
+
+def _diag_route(
+ index_url: str,
+ *,
+ skip_torch: bool = False,
+ backend: "str | None" = None,
+ nvidia: bool = False,
+) -> bool:
+ """Whether install.sh routes the two node diagnoses for this wheel index. Lifted rather
+ than restated, since the thing under test is which patterns the case actually lists."""
+ lines = _install_sh_lines()
+ start = next(i for i, line in enumerate(lines) if line.startswith("_amd_node_diag_leaf="))
+ esac_at = next(i for i in range(start, len(lines)) if lines[i] == "esac")
+ # The --no-torch override and the explicit-backend case below it are part of the same
+ # decision, so the span runs to the end of both rather than stopping at the first esac.
+ _skip_torch_end = next(i for i in range(esac_at, len(lines)) if lines[i] == "fi")
+ end = next(i for i in range(_skip_torch_end, len(lines)) if lines[i] == "esac")
+ script = "\n".join(
+ [
+ f"TORCH_INDEX_URL={index_url!r}",
+ f"SKIP_TORCH={'true' if skip_torch else 'false'}",
+ # Set either way, so the arms below do not inherit whatever the runner exports.
+ f"export UNSLOTH_LLAMA_CPP_BACKEND={backend or ''!r}",
+ # The classifiers, not stubs: which leaves count as a ROCm route is exactly what
+ # these tests are about. _requested_llama_backend comes with them because every
+ # scope predicate reads the backend request through it.
+ _shell_fn(lines, "_torch_index_url_leaf"),
+ _shell_fn(lines, "_is_pip_rocm_family_leaf"),
+ _shell_fn(lines, "_requested_llama_backend"),
+ _shell_fn(lines, "_torch_opens_amd_nodes"),
+ # Stubbed: the real one runs nvidia-smi and would answer from the runner's own
+ # hardware. False by default, so each arm reads as an AMD-only host.
+ f"_has_usable_nvidia_gpu() {{ return {0 if nvidia else 1}; }}",
+ _shell_fn(lines, "_auto_bundle_opens_amd_nodes"),
+ _shell_fn(lines, "_run_may_open_a_gpu_node"),
+ *lines[start : end + 1],
+ 'echo "$_amd_node_diag_route"',
+ ]
+ )
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, check = True)
+ return out.stdout.strip() == "true"
+
+
[email protected]("index_url, routed", [
+ # The two arms this installer prints a wheel line for are the two the diagnoses belong
+ # to, and #10466's host reaches the second by reroute rather than the first.
+ ("https://download.pytorch.org/whl/cpu", True),
+ ("https://download.pytorch.org/whl/rocm7.0", True),
+ ("https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0/gfx1151", True),
+ # Moving the diagnoses out of the */cpu arm let them reach an index the case has no
+ # arm for at all. _has_amd_rocm_gpu returns false on ANY host with a usable NVIDIA
+ # GPU, so on a CUDA route its condition is satisfied by every hybrid box with an AMD
+ # card on the bus, and someone correctly installing CUDA wheels was told to install
+ # the ROCm kernel stack for a card this install does not use.
+ ("https://download.pytorch.org/whl/cu128", False),
+ ("https://download.pytorch.org/whl/xpu", False),
+])
+def test_the_node_diagnoses_run_on_the_routes_the_case_reports(index_url, routed):
+ """Which wheel routes reach the two node diagnoses at all."""
+ assert _diag_route(index_url) is routed
+
+
+def test_the_installer_makes_the_same_owner_versus_external_distinction(tmp_path):
+ """Read off install.sh, since the two halves must agree: the awk owner branch printed
+ owner: for every node this account owns, so a node whose OWNER digit already grants rw got
+ the mode advice there too. Lifted rather than restated, so a revert fails here."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+
+ node.chmod(0o600) # owner rw: the mode is not what is shutting it
+ out = _install_sh_hint(str(node), self_uid = str(os.getuid()), repairs = None)
+ assert "granted read and write by the" in out
+ # Never by naming the owner: the same bucket carries other-class nodes this account
+ # does not own, and one wording has to be true of both.
+ assert "is owned by this account and its owner bits" not in out
+ # The installer wraps one sentence over several substep lines, so match a fragment
+ # that cannot straddle the break.
+ assert "device cgroup or an LSM" in out
+ assert "fix the mode" not in out
+
+ node.chmod(0o060) # owner has nothing: the mode IS the repair, and still is
+ out = _install_sh_hint(str(node), self_uid = str(os.getuid()), repairs = None)
+ assert "fix the mode" in out
+ assert "already grant read and write" not in out
+
+
[email protected]("mode, bucket", [
+ # Codex 4041533207. os.access said the node is shut, so on a node this account owns
+ # whose OWNER bits already read rw, the mode is not what denies it: an LSM or a
+ # container device policy is, and a mode change or a udev rule repairs nothing. It was
+ # filed as an owner-mode problem and prescribed exactly that.
+ pytest.param(0o600, "external", id = "owner_bits_that_already_grant_it_are_external"),
+ # The control, and the owner-precedence rule itself: POSIX resolves the owner class
+ # exclusively once the uid matches, so a node this account owns whose owner bits deny
+ # cannot be opened by joining its group however the group bits read. The mode is the
+ # repair, and usermod there is a command that succeeds and changes nothing.
+ pytest.param(0o060, "owned", id = "a_node_this_account_owns_is_not_answered_with_a_group"),
+])
+def test_how_a_node_this_account_owns_is_classified(monkeypatch, tmp_path, mode, bucket):
+ """Which bucket an owned node lands in, by owner bits. Every other bucket must stay empty:
+ a group named here is a repair that cannot work."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(mode)
+ monkeypatch.setattr(amd, "_has_an_access_acl", lambda _path: False)
+ assert amd._groups_that_own([str(node)]) == _buckets(**{bucket: [str(node)]})
+
+
+# fmt: off
[email protected]("mode, in_the_group, bucket", [
+ # Codex 4049299005. Neither the owner nor in the owning group puts this account in the
+ # OTHER class, which POSIX resolves exclusively just as it does the owner one: bits that
+ # already grant rw mean the mode is not what denies a node os.access() called shut, so
+ # neither a chmod nor a usermod repairs it. It was classified from the group bits and
+ # answered with a group to join.
+ pytest.param(0o666, False, "external",
+ id = "other_bits_that_already_grant_it_are_external"),
+ # The control, or the branch would be "never name a group", which removes a correct
+ # repair: with the other bits denying, joining the group really would open the node.
+ pytest.param(0o660, False, "joinable",
+ id = "other_bits_that_deny_still_leave_a_group_worth_joining"),
+ # And membership outranks the other class, because a member is in the GROUP class and
+ # the other bits are never consulted there. `already` names the group rather than
+ # filing the node under a bucket that names nothing.
+ pytest.param(0o666, True, "already",
+ id = "a_member_is_still_told_the_group_it_already_holds"),
+])
+# fmt: on
+def test_how_a_node_this_account_neither_owns_nor_shares_a_group_with_is_classified(
+ monkeypatch, tmp_path, mode, in_the_group, bucket
+):
+ """Which bucket the third permission class lands in. The two halves of this rule were
+ already written for the owner class and for a group this account holds; this is the same
+ question for the one class that was left reading its neighbour's bits."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(mode)
+ _gid = node.stat().st_gid
+ monkeypatch.setattr(amd, "_has_an_access_acl", lambda _path: False)
+ # Not the owner, so the owner class cannot claim the node first. Read BEFORE the patch
+ # lands: amd.os is the os module itself, so a lambda calling os.getuid() would call the
+ # replacement and recurse until the stack ends.
+ _not_the_owner = os.getuid() + 1
+ monkeypatch.setattr(amd.os, "getuid", lambda: _not_the_owner)
+ _held = {_gid} if in_the_group else {_gid + 10_000}
+ monkeypatch.setattr(amd.os, "getgid", lambda: sorted(_held)[0])
+ monkeypatch.setattr(amd.os, "getgroups", lambda: sorted(_held))
+
+ joinable, unnamed, no_group, acl, owned, privileged, already, external = (
+ amd._groups_that_own([str(node)])
+ )
+ _by_name = dict(
+ joinable = joinable, unnamed = unnamed, no_group = no_group, acl = acl,
+ owned = owned, privileged = privileged, already = already, external = external,
+ )
+ assert _by_name[bucket], f"{bucket} is empty: {_by_name}"
+ for _name, _got in _by_name.items():
+ if _name != bucket:
+ assert not _got, f"{_name} must stay empty, got {_got}"
+
+
+def test_the_installer_makes_the_same_other_class_distinction(tmp_path):
+ """The shell half of the rule above, read off install.sh so the two cannot drift: the awk
+ classifier reached its group digit for a node in the other class too, so mode 0666 printed
+ a group to join. A revert fails here as well as in the Python case."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ _not_us = str(os.getuid() + 1)
+
+ node.chmod(0o666) # other rw: no membership and no mode change opens this
+ out = _install_sh_hint(str(node), self_uid = _not_us, repairs = None)
+ assert "device cgroup or an LSM" in out
+ assert "usermod" not in out
+
+ node.chmod(0o660) # other denies: the group really is the repair
+ out = _install_sh_hint(str(node), self_uid = _not_us, repairs = None)
+ assert "device cgroup or an LSM" not in out
+
+
+def test_a_node_carrying_an_acl_is_not_answered_with_usermod(monkeypatch, tmp_path):
+ """acl(5): once an access ACL is present, the group-class bits in st_mode are the ACL MASK
+ rather than the owning group's grant, so a node whose mask reads rw can still deny its
+ group. Prescribing membership from the mode there is a promise the stat cannot support."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(0o660)
+ monkeypatch.setattr(amd, "_has_an_access_acl", lambda path: True)
+ _not_the_owner = os.getuid() + 1
+ monkeypatch.setattr(amd.os, "getuid", lambda: _not_the_owner)
+ joinable, unnamed, no_group, acl, owned, _priv, _already, _ext = amd._groups_that_own(
+ [str(node)]
+ )
+ assert acl == [str(node)]
+ assert joinable == [] and unnamed == [] and no_group == []
+
+
+ pytest.param(0o660, id = "the_same_node_without_an_acl_is_still_prescribed_for"),
+ pytest.param(0o060, id = "the_same_node_owned_by_someone_else_is_still_a_group"),
+])
+def test_an_ordinary_node_owned_elsewhere_is_still_prescribed_for(monkeypatch, tmp_path, mode):
+ """The control for both suppressions above: the ordinary node, whose mode bits ARE the
+ group's grant and whose owner is somebody else, so the owner class does not apply. Without
+ it the rule could decline to prescribe anywhere, which removes the repair #10466 needs."""
+ node, _group = _a_node_a_membership_would_open(tmp_path, mode = mode)
+ # The fixture can only chgrp to a group this account holds, and a node whose owning group
+ # the account already has is filed under `already` rather than `joinable`. These arms are
+ # about the derivation, so stand the account outside that group.
+ _not_my_group = os.getgid() + 1
+ monkeypatch.setattr(amd.os, "getgid", lambda: _not_my_group)
+ monkeypatch.setattr(amd.os, "getgroups", lambda: [])
+ monkeypatch.setattr(amd, "_has_an_access_acl", lambda path: False)
+ _not_the_owner = os.getuid() + 1
+ monkeypatch.setattr(amd.os, "getuid", lambda: _not_the_owner)
+ joinable, unnamed, no_group, acl, owned, privileged, _already, _ext = amd._groups_that_own(
+ [str(node)]
+ )
+ assert acl == [] and owned == []
+ assert joinable or unnamed
+
+
+def test_the_installer_reports_an_acl_rather_than_prescribing_membership(tmp_path):
+ """The shell twin of the same rule: ls marks such a node with a trailing "+", which is the
+ marker available without getfacl."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(0o660)
+ try:
+ _set = subprocess.run(["setfacl", "-m", "u:nobody:rw", str(node)], capture_output = True)
+ except OSError:
+ pytest.skip("setfacl is not installed")
+ if _set.returncode != 0:
+ pytest.skip("this filesystem does not support ACLs")
+ out = _install_sh_hint(str(node))
+ assert "carries a POSIX ACL" in out
+ assert "usermod -a -G" not in out
+
+
+def test_the_installer_says_the_missing_render_node_with_nothing_closed():
+ """The installer's half of the container case: nothing closed, no render node, and an AMD
+ GPU in the KFD topology."""
+ out = _install_sh_hint("", render_present = False, amd_present = True)
+ assert "no AMD render node" in out
+ assert "--device /dev/dri" in out
+
+
+def test_the_installer_stays_quiet_on_a_host_with_no_amd_gpu():
+ """Its control, and the same vendor trap: without the KFD topology test this fires on
+ every host whose /dev/dri holds another vendor's nodes, or none at all."""
+ out = _install_sh_hint("", render_present = False, amd_present = False)
+ assert out.strip() == ""
+
+
+ # The probe on a file with none: it has to answer False for the common node or every
+ # host stops being prescribed for.
+ pytest.param((True, None, False), id = "an_ordinary_node_reports_no_acl"),
+ # And the failure mode that must not raise: this runs where things are already wrong,
+ # so an unreadable path answers False rather than taking the hint down.
+ pytest.param((False, None, False), id = "a_path_that_cannot_be_read_reports_no_acl"),
+ # os.listxattr returns ``str`` names for a ``str`` path, so the bytes literal this
+ # first shipped with could never match one and the whole ACL branch was dead. Stubbed
+ # rather than written with setfacl, which is not installed here: the shell twin above
+ # skips when it is absent, and a probe whose positive direction is only ever exercised
+ # by a skipping test is not exercised at all. That is how this survived a round.
+ pytest.param((True, ["security.selinux", "system.posix_acl_access"], True),
+ id = "the_acl_probe_matches_the_name_type_listxattr_returns"),
+ # A bytes path yields bytes names, and the caller chooses the path type, so both are
+ # accepted. The control for the case above: without it, swapping one literal for the
+ # other passes just as well and nothing says which type is actually returned.
+ pytest.param((True, [b"system.posix_acl_access"], True),
+ id = "the_acl_probe_also_reads_bytes_names"),
+ # The negative control. An ACL is claimed from one exact name, so a node carrying only
+ # other attributes stays prescribed for.
+ pytest.param((True, ["security.selinux", "user.note"], False),
+ id = "another_xattr_is_not_read_as_an_acl"),
+])
+def test_what_the_acl_probe_answers(monkeypatch, tmp_path, case):
+ """The probe the bucket rule above consults."""
+ create, xattrs, carries_one = case
+ node = tmp_path / "renderD128"
+ if create:
+ node.write_bytes(b"")
+ if xattrs is not None:
+ # raising=False: os.listxattr does not exist on macOS or Windows, so without it
+ # monkeypatch fails on the PATCH rather than the probe, and the case never runs.
+ # _has_an_access_acl already answers False through AttributeError there, which is
+ # what these cases assert about a host that cannot report an ACL.
+ monkeypatch.setattr(amd.os, "listxattr", lambda path: xattrs, raising = False)
+ assert amd._has_an_access_acl(str(node)) is carries_one
+
+
+def _vulkan_reason_with_open_sibling(monkeypatch, openable: set) -> str:
+ """The empty-probe reason for a Vulkan build with renderD128 closed."""
+ _nodes(
+ monkeypatch,
+ present = ["/dev/dri/renderD128", "/dev/dri/renderD129"],
+ openable = openable,
+ )
+ for var in _GPU_MASK_VARS:
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_installed_ggml_backends", staticmethod(lambda _b: frozenset({"vulkan"}))
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda _b: True))
+ return _empty_probe()
+
+
+_VULKAN_REASON = "the Vulkan probe reported no device"
+
+
+ # A closed node explains an empty probe only when it is the node the runtime would
+ # have used. With renderD129 open the loader had one to enumerate and still reported
+ # nothing, so the closed renderD128 is a second finding; returning it alone sends the
+ # user after a repair that leaves the probe just as empty. It is still said, because
+ # it is still true. Fails before the fix, which returned the hint unconditionally.
+ pytest.param(({"/dev/dri/renderD129"}, (_VULKAN_REASON, "/dev/dri/renderD128"), ()),
+ id = "an_open_sibling_node_keeps_the_vulkan_reason"),
+ # The control, and the #10466 host itself: with every AMD node closed there is no
+ # sibling the loader could have used, so the closed node IS the reason and must not be
+ # demoted to a footnote behind a Vulkan sentence that explains nothing.
+ pytest.param((set(), ("/dev/dri/renderD128",), (_VULKAN_REASON,)),
+ id = "no_open_sibling_still_gives_the_node_hint_alone"),
+])
+def test_whether_a_vulkan_sibling_node_answers_the_empty_probe(monkeypatch, linux, case):
+ """Whether a closed render node is the reason, given what the loader could still open."""
+ openable, says, does_not_say = case
+ reason = _vulkan_reason_with_open_sibling(monkeypatch, openable)
+ _asserts(reason, says, does_not_say)
+
+
+def _hip_reason_with_nodes(monkeypatch, present: list, openable: set) -> str:
+ """The empty-probe reason for a HIP build over a given node layout."""
+ _nodes(monkeypatch, present = present, openable = openable)
+ for var in _GPU_MASK_VARS:
+ monkeypatch.delenv(var, raising = False)
+ monkeypatch.setattr(
+ LlamaCppBackend, "_installed_ggml_backends", staticmethod(lambda _b: frozenset({"hip"}))
+ )
+ monkeypatch.setattr(LlamaCppBackend, "_is_vulkan_backend", staticmethod(lambda _b: False))
+ return _empty_probe()
+
+
+_SEPARATELY = "Separately, and not why the probe is empty"
+
+
+ # ROCm needs /dev/kfd and a render node. With both open on a multi-AMD host, a closed
+ # SECOND render node is not why the probe came back empty, and returning the group
+ # repair as the sole diagnosis leaves the user fixing something that changes nothing.
+ # Fails before the fix, which asked the sibling question of Vulkan builds only.
+ pytest.param(({"/dev/kfd", "/dev/dri/renderD129"}, (_SEPARATELY, "/dev/dri/renderD128"), ()),
+ id = "a_closed_sibling_beside_an_open_rocm_path_is_not_the_reason"),
+ # The control that keeps it narrow: /dev/kfd has no sibling, so a closed one blocks
+ # ROCm outright however many render nodes are open.
+ pytest.param(({"/dev/dri/renderD129"}, ("/dev/kfd",), (_SEPARATELY,)),
+ id = "a_closed_kfd_is_still_the_reason_for_a_hip_build"),
+])
+def test_whether_a_hip_sibling_node_answers_the_empty_probe(monkeypatch, linux, case):
+ """The same sibling question asked of a ROCm build, over one three-node host."""
+ openable, says, does_not_say = case
+ reason = _hip_reason_with_nodes(
+ monkeypatch,
+ present = ["/dev/kfd", "/dev/dri/renderD128", "/dev/dri/renderD129"],
+ openable = openable,
+ )
+ _asserts(reason, says, does_not_say)
+
+
[email protected]("layout, blocks", [
+ # --device /dev/kfd without --device /dev/dri. The one node mapped opens, so nothing
+ # is CLOSED, and this answered False -- which made hardware.py suppress the very hint
+ # that names the repair, and llama_cpp.py file it as "not why the probe is empty" when
+ # the absent render node is exactly why.
+ pytest.param(_KFD_OPEN, True, id = "a_missing_render_node_blocks_the_runtime"),
+ # The control. Without it the fix could be "always blocks", which suppresses nothing
+ # and labels every empty probe a permission problem.
+ pytest.param(_BOTH_OPEN, False, id = "a_complete_open_mapping_still_does_not_block"),
+])
+def test_whether_the_node_layout_blocks_the_runtime(monkeypatch, linux, layout, blocks):
+ """The predicate both callers gate their hints on."""
+ _nodes(monkeypatch, **layout)
+ assert amd.amd_closed_nodes_block_the_runtime() is blocks
+
+
+def test_the_installer_does_not_dangle_the_group_sentence(tmp_path):
+ """When every refused node has an unnamed GID, an ACL, or a mode no group can open,
+ _amd_node_repairs names no group on purpose. The installer printed "Add yourself to the"
+ above that branch regardless, so the message read as an instruction cut off mid-sentence
+ and then contradicted."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(0o600) # owner-only: no membership opens it, so no group is named
+ out = _install_sh_hint(str(node))
+ assert "Add yourself to the" not in out
+ assert "no" in out and "membership opens it" in out
+
+
+# fmt: on
+def _says(
+ out: str,
+ contains,
+ absent = (),
+):
+ """Assert what a lifted message says; ``contains = None`` means it says nothing at all."""
+ if contains is None:
+ assert out.strip() == "", out
+ return
+ for _text in contains:
+ assert _text in out, _text
+ for _text in absent:
+ assert _text not in out, _text
+
+
+def _cases(*rows):
+ """Parametrize rows written id first, so one case of a family reads as one line."""
+ return [pytest.param(*row[1:], id = row[0]) for row in rows]
+
+
+def test_the_installer_stops_at_the_owner_class_too(tmp_path):
+ """The shell half of the owner-precedence item: with the caller as the owner, the
+ installer must not print a usermod line for a node no membership opens."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(0o060)
+ out = _install_sh_hint(str(node), self_uid = str(os.getuid()))
+ assert "usermod" not in out
+ assert "owned by this account" in out
+
+
+_SEEN = dict(topology = True, amd_smi_sees_it = True)
+_NO_TORCH = {**_SEEN, "skip_torch": True, "backend": None}
+_BLIND = dict(topology = False, amd_smi_sees_it = False)
+_SEEING = dict(topology = False, amd_smi_sees_it = True)
+_STACK = "Install the ROCm kernel stack"
+
+
+def _kernel_stack_hint_text(
+ tmp_path,
+ *,
+ topology: bool,
+ kfd_present: bool = False,
+ topology_readable: bool = True,
+ nvidia: bool = False,
+) -> str:
+ """What install.sh actually PRINTS in the missing-/dev/kfd branch, with ROCm blind.
+
+ `_kernel_stack_hint_runs` above lifts only the guard, so it answers whether the branch
+ fires and nothing about which repair it names, which is exactly where the branch was
+ wrong. _install_sh_missing_kfd below lifts the guard AND its body, through the closing
+ `fi`, so a revert changes the text this returns; this name is the ROCm-sees-nothing
+ corner of it, which is the shape the kernel-stack advice is written for.
+ """
+ return _install_sh_missing_kfd(
+ tmp_path,
+ topology = topology,
+ kfd_present = kfd_present,
+ topology_readable = topology_readable,
+ amd_smi_sees_it = False,
+ nvidia = nvidia,
+ )
+
+
+# fmt: off
[email protected]("topology, contains, absent", _cases(
+ ("a_topology_means_the_driver_is_loaded", True, ["--device /dev/kfd", "the node itself"],
+ [_STACK]),
+ ("no_topology_keeps_the_kernel_stack_advice", False, [_STACK], ["--device /dev/kfd"]),
+))
+# fmt: on
+def test_the_installers_missing_kfd_repair_follows_the_topology(tmp_path, topology, contains, absent):
+ """A container created with --device /dev/dri and no --device /dev/kfd sees the host's
+ /sys and not its /dev, so the KFD topology names an AMD GPU while the node is absent: the
+ driver is loaded, and "install the ROCm kernel stack" leaves HIP as unavailable as before.
+ No topology is the case the branch was written for, the driver really is missing, and
+ without that arm the fix could be "never mention the kernel stack", which removes a
+ correct diagnosis."""
+ _says(_kernel_stack_hint_text(tmp_path, topology = topology), contains, absent)
+
+
+def test_a_container_missing_kfd_is_told_to_map_it_rather_than_reinstall(monkeypatch, linux):
+ """The runtime half of the same item. `_amd_nodes_the_runtime_lacks` reports a missing
+ node only once the KFD topology names an AMD GPU, and that topology is the amdkfd
+ driver's own sysfs, so on every host this sentence can reach the kernel stack is already
+ loaded and the advice to install it is unreachable-by-construction wrong."""
+ _nodes(monkeypatch, present = ["/dev/dri/renderD128"], openable = {"/dev/dri/renderD128"})
+ hint = amd.amd_node_permission_hint()
+ assert "--device /dev/kfd" in hint
+ assert "kernel stack" not in hint
+ assert "the kernel driver is loaded" in hint
+
+
+def _install_sh_missing_kfd(
+ tmp_path,
+ *,
+ topology: bool,
+ kfd_present: bool = False,
+ topology_readable: bool = True,
+ confirmed_drm: bool = False,
+ amd_smi_sees_it: "bool | None" = None,
+ rocm_visible: "bool | None" = None,
+ skip_torch: bool = False,
+ backend: "str | None" = None,
+ nvidia: bool = False,
+) -> str:
+ """What the installer says when /dev/kfd is absent, for a given pair of probes.
+
+ Lifts the two branches together, through the closing `fi`, because which of them runs
+ is the thing under test. A test operator cannot be stubbed, so `[ -e /dev/kfd ]` is
+ kept as an `-e` test on a path this case owns: the operator still runs, and what it
+ runs against is `kfd_present` rather than whatever the machine happens to have.
+
+ This used to skip on a host that has /dev/kfd, which meant the whole family ran on dev
+ boxes with no AMD GPU and vanished on the hardware it is about -- and its mirror at
+ `test_the_installer_names_the_userspace_when_the_node_is_already_there` skipped
+ everywhere else, so no single host ever ran both arms.
+ """
+ lines = _install_sh_lines()
+ # Anchored on the kernel-stack SENTENCE, then walked back to the `if` above it, since
+ # neither branch's condition is stable enough to anchor on: the mapping one is what an
+ # earlier change edited, and a revert that stopped the extraction finding anything would
+ # read "the text changed" as "the behaviour changed". The predicate _amd_gpu_present_via_pci
+ # is named twice in this installer, so it is not an anchor either.
+ end = _install_sh_anchor(lines, _PCI_SENTENCE)
+ start = _install_sh_if_above(lines, end)
+ close = next(i for i in range(end + 1, len(lines)) if lines[i] == "fi")
+ script = "\n".join(
+ [
+ 'substep() { echo "$1"; }',
+ 'C_WARN=""',
+ f"SKIP_TORCH={'true' if skip_torch else 'false'}",
+ "OS=linux",
+ "_amd_node_diag_route=true",
+ *_run_scope_defs(lines, nvidia = nvidia),
+ f"_kfd_topology_has_an_amd_gpu() {{ return {0 if topology else 1}; }}",
+ # The branch asks _amd_silicon_behind_a_missing_kfd, not the topology directly,
+ # so the real one is LIFTED and only its inputs are stubbed: a copy here would
+ # answer for itself and the container fallback would go untested.
+ # State 0 names AMD, 1 read it and found none, 2 could not read it at all.
+ f"_kfd_topology_amd_state() {{ return {0 if topology else (1 if topology_readable else 2)}; }}",
+ f"_a_confirmed_amd_render_node_exists() {{ return {0 if confirmed_drm else 1}; }}",
+ _shell_fn(lines, "_amd_silicon_behind_a_missing_kfd"),
+ _shell_fn(lines, "_kfd_node_is_amds"),
+ # Stubbed when the arm is about something else and only needs a verdict; run
+ # for real over stubbed command lookups when the arm IS about which probe the
+ # branch consults, since a stub of the probe under test would answer for it.
+ # Running it for real also exercises the "ignore-nvidia" argument the branch
+ # passes, which is what keeps the diagnosis off the NVIDIA short-circuit.
+ *(
+ [f"_has_amd_rocm_gpu() {{ return {0 if amd_smi_sees_it else 1}; }}"]
+ if rocm_visible is None
+ else [
+ "_ensure_rocm_probe_env() { :; }",
+ 'command() { case "$2" in rocminfo) return 0 ;; *) return 1 ;; esac; }',
+ "rocminfo() { echo ' Name: gfx1151'; }"
+ if rocm_visible
+ else "rocminfo() { return 1; }",
+ _shell_fn(lines, "_has_amd_rocm_gpu"),
+ ]
+ ),
+ "_amd_gpu_present_via_pci() { return 0; }",
+ "\n".join(lines[start : close + 1]),
+ ]
+ )
+ script = _kfd_node_the_case_owns(script, tmp_path, present = kfd_present)
+ env = {**os.environ, "_closed_amd_nodes": ""}
+ env.pop("UNSLOTH_LLAMA_CPP_BACKEND", None)
+ if backend is not None:
+ env["UNSLOTH_LLAMA_CPP_BACKEND"] = backend
+ out = subprocess.run(
+ ["bash", "-c", script], capture_output = True, text = True, check = True, env = env
+ )
+ return out.stdout
+
+
+_ABSENT_KFD = "/dev/kfd is not present"
+
+
+# fmt: off
[email protected]("kwargs, contains, absent", _cases(
+ ("amd_smi_does_not_suppress_the_mapping_advice", _SEEN, ["--device /dev/kfd"], [_STACK]),
+ ("no_topology_and_a_blind_rocm_keeps_the_advice", _BLIND, [_STACK], []),
+ ("no_topology_but_a_seeing_amd_smi_withholds_it", _SEEING, [], [_STACK]),
+ ("a_torch_install_still_gets_the_missing_kfd_advice", _SEEN, [_ABSENT_KFD], []),
+ ("a_no_torch_rocm_bundle_is_told_its_kfd_is_missing", _NO_TORCH, [_ABSENT_KFD], []),
+ ("a_no_torch_vulkan_run_is_not_told_about_it", {**_NO_TORCH, "backend": "vulkan"}, None, []),
+ ("a_no_torch_cuda_run_is_not_told_about_it_either", {**_NO_TORCH, "backend": "cuda"}, None, []),
+ # Codex 4049894549, and the container this whole feature was written for: --device
+ # /dev/dri with no --device /dev/kfd, and /sys/class/kfd masked with it. The topology
+ # cannot be READ, so gating on it alone left this host with no diagnosis at all -- the
+ # two PCI branches below need `! _has_amd_rocm_gpu`, and amd-smi answers through libdrm
+ # here, while a node that does not EXIST can never reach the closed-node list. A
+ # confirmed AMD render node is the standing evidence, exactly as the Python half reads
+ # it. Fails without the fallback.
+ ("an_unreadable_topology_with_a_confirmed_amd_render_node",
+ {"topology": False, "amd_smi_sees_it": True, "topology_readable": False,
+ "confirmed_drm": True}, ["--device /dev/kfd"], [_STACK]),
+ # The control that keeps it from becoming "always advise": same masked sysfs, but no
+ # AMD render node confirmed, so there is no evidence of AMD silicon and naming one
+ # would invent the finding.
+ ("an_unreadable_topology_with_no_confirmed_node_stays_silent",
+ {"topology": False, "amd_smi_sees_it": True, "topology_readable": False,
+ "confirmed_drm": False}, None, ["--device /dev/kfd"]),
+ # And the vendor really is read: a readable topology that names no AMD is not rescued
+ # by the fallback, or an NVIDIA-only host would claim the card.
+ ("a_readable_topology_naming_no_amd_is_not_rescued_by_drm",
+ {"topology": False, "amd_smi_sees_it": True, "topology_readable": True,
+ "confirmed_drm": True}, None, ["--device /dev/kfd"]),
+))
+# fmt: on
+def test_what_the_installer_says_when_the_kfd_node_is_absent(tmp_path, kwargs, contains, absent):
+ """amd-smi reads the driver over sysfs and libdrm, so it lists the card in a container
+ given only --device /dev/dri, where HIP has no /dev/kfd to open; llama_cpp.py's
+ _rocm_hip_is_reachable documents that disagreement. Behind _has_amd_rocm_gpu the mapping
+ advice was suppressed on the container shape it was written for, and nothing else spoke:
+ the render node is open, so no node is closed and none is missing. The two are separate
+ branches rather than one with an inner test because they need different evidence -- with
+ no KFD topology the driver really is missing, and that diagnosis is still gated on ROCm
+ seeing nothing, so a host whose amd-smi answers is not told to install what it has.
+
+ The missing-node report was gated on SKIP_TORCH=false, so a --no-torch run whose GGUF
+ bundle is ROCm -- which opens /dev/kfd exactly as torch would -- finished silently in a
+ container mapping /dev/dri and not /dev/kfd: nothing closed, nothing missing said, no
+ account of why the backend cannot initialise. A Vulkan or CUDA bundle opens no /dev/kfd,
+ so its absence explains nothing about them and the run stays silent, and the ordinary
+ torch install is unchanged by the gate swap."""
+ _says(_install_sh_missing_kfd(tmp_path, **kwargs), contains, absent)
+
+
+# fmt: off
[email protected]("assignment, needs", _cases(
+ ("the_closed_node_read", '_closed_amd_nodes="$(_amd_nodes_closed_to_this_user',
+ ("stat", "awk")),
+ ("the_index_leaf_read", "_amd_node_diag_leaf=$(_torch_index_url_leaf", ("tr",)),
+ ("the_repair_classifier", "_closed_amd_repairs=$(_amd_node_repairs", ("awk",)),
+))
+# fmt: on
+def test_a_diagnostic_cannot_take_the_install_down_with_it(assignment, needs):
+ """install.sh runs under `set -e` from line 5, so an unguarded command substitution
+ aborts the whole installer when its helper's last command fails. These three shell out
+ to stat, awk and tr, none of which is guaranteed on a minimal container, and two of
+ them run on every Linux install whether or not anything is closed.
+
+ The answer on such a host has to be "no advice", never "no install": the diagnostic
+ exists to explain a GPU that is not working, and an install that dies instead is
+ strictly worse than the silence it replaced. Every consumer already reads empty as
+ nothing to report.
+
+ Verified live rather than by reading: with tr stubbed to exit 127 the unguarded form
+ aborts at the leaf read before installing anything, and the guarded form continues.
+ Asserted on the source rather than by running the installer, because reaching these
+ lines for real means running an install."""
+ line = next(_l for _l in _install_sh_lines() if _l.lstrip().startswith(assignment))
+ assert "|| true" in line, f"{assignment} is unguarded under set -e: {line.strip()}"
+
+
+# fmt: off
[email protected]("helper, tools", _cases(
+ ("the_closed_node_read_survives", "_amd_nodes_closed_to_this_user", ("stat", "awk", "tr")),
+ ("the_repair_classifier_survives", "_amd_node_repairs", ("stat", "awk", "tr")),
+ ("the_index_leaf_read_survives", "_torch_index_url_leaf", ("stat", "awk", "tr")),
+))
+# fmt: on
+def test_the_guard_actually_holds_when_those_tools_are_missing(helper, tools):
+ """The half that makes the test above mean something: the `|| true` is only worth
+ asserting if the assignment really does survive. Every named tool is stubbed to exit
+ 127, which is what a minimal container looks like, and the script must reach its last
+ line with status 0."""
+ lines = _install_sh_lines()
+ _lifted, _seen, _queue = [], set(), [helper]
+ while _queue:
+ _name = _queue.pop(0)
+ if _name in _seen:
+ continue
+ _seen.add(_name)
+ try:
+ _body = _shell_fn(lines, _name)
+ except StopIteration:
+ continue
+ _lifted.append(_body)
+ _queue += sorted(set(re.findall(r"\b(_[a-z0-9_]+)\b", _body)) - _seen)
+ script = "\n".join(
+ [
+ *[f"{_t}() {{ return 127; }}" for _t in tools],
+ *_lifted,
+ "set -e",
+ f'_answer="$({helper} /dev/kfd || true)"',
+ 'echo "REACHED THE END"',
+ ]
+ )
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True)
+ assert out.returncode == 0, out.stderr
+ assert "REACHED THE END" in out.stdout
+
+
+def test_the_installer_denies_the_same_groups_the_runtime_does():
+ """The two lists are maintained by hand in two languages, so drift is the failure mode.
+ Read install.sh's alternation and compare it to the constant rather than restating
+ either: a group added to one half alone fails here."""
+ install_sh = Path(__file__).resolve().parents[3] / "install.sh"
+ text = install_sh.read_text(encoding = "utf-8")
+ match = re.search(r"gname ~ /\^\(([a-z|]+)\)\$/", text)
+ assert match, "install.sh no longer carries the privileged-group alternation"
+ assert set(match.group(1).split("|")) == set(amd._PRIVILEGED_GROUPS)
+
+
+_TWO_GIDS = "gid:993\ngid:994"
+_BOTH_FLAGS = "--group-add 993 --group-add 994"
+_RENDER = "/dev/dri/renderD128"
+
+
+# fmt: off
[email protected]("closed_nodes, repairs, contains, absent", _cases(
+ ("the_container_flag_is_repeated_for_every_gid", f"/dev/kfd\n{_RENDER}", _TWO_GIDS,
+ [_BOTH_FLAGS, "GIDs 993,994"], []),
+ ("one_gid_still_reads_as_one", "/dev/kfd", "gid:993", ["--group-add 993", "GID 993"],
+ ["--group-add 993 --group-add"]),
+ ("the_bare_host_repair_also_adds_the_account", _RENDER, _TWO_GIDS,
+ ["sudo groupadd -g 993 amdgpu993", "sudo usermod -a -G amdgpu993 ada",
+ "sudo groupadd -g 994 amdgpu994", "sudo usermod -a -G amdgpu994 ada", _BOTH_FLAGS,
+ "create a group for each"], []),
+ ("and_says_to_start_a_new_session", _RENDER, "gid:993",
+ ["sudo groupadd -g 993 amdgpu993", "log out and back in"], []),
+))
+# fmt: on
+def test_the_installers_unnamed_gid_repair(closed_nodes, repairs, contains, absent):
+ """--group-add takes a SINGLE value, so a comma-joined pair is one group name that does
+ not exist, and naming only the first leaves the second node shut. docker/run.sh repeats
+ the flag and the Python half already emits it repeated; the installer said "the numeric
+ GID", singular, for a value it had just printed as "993,994". The singular wording still
+ has to survive for one GID, so the fix is not "always say GIDs". groupadd gives the
+ numeric owner a NAME and does not put this account in the group, so the bare-host repair
+ needs the usermod too -- the installer printed the container flags per GID after the
+ earlier fix but still said only "create a group" -- and that usermod needs the same
+ session refresh the named-group branch ends with, in its own sentence, since this fixture
+ never reaches the named-group branch."""
+ _says(_install_sh_hint(closed_nodes, repairs = repairs), contains, absent)
+
+
+_HIP_1_SURVIVES = ["HIP_VISIBLE_DEVICES='1'", "which the groups do not clear"]
+
+
+# fmt: off
[email protected]("env, contains, absent", _cases(
+ ("an_empty_token_keeps_the_prefix_it_already_counted",
+ {"ROCR_VISIBLE_DEVICES": "0,", "HIP_VISIBLE_DEVICES": "1"}, _HIP_1_SURVIVES, []),
+ ("a_repeated_ordinal_surfaces_one_device",
+ {"ROCR_VISIBLE_DEVICES": "0,0", "HIP_VISIBLE_DEVICES": "1"}, _HIP_1_SURVIVES, []),
+ ("an_illegal_selector_is_reported_as_a_blocker", {"ROCR_VISIBLE_DEVICES": "garbage"},
+ ["ROCR_VISIBLE_DEVICES"], ["cannot resolve"]),
+ ("a_uuid_selector_is_still_only_unresolved", {"ROCR_VISIBLE_DEVICES": "GPU-4b2c9f1e0a7d3b58"},
+ ["cannot resolve"], []),
+))
+# fmt: on
+def test_how_a_rocr_selector_reads_on_a_two_gpu_host(monkeypatch, linux, env, contains, absent):
+ """ROCr's RvdFilter builds its list from tokens that are "Legal and NOT Terminating", so
+ an entry it cannot evaluate ends the list and the devices BEFORE it still survive:
+ ROCR_VISIBLE_DEVICES='0,' leaves one device and HIP ordinal 1 hides it, which read as an
+ unresolvable list left the HIP mask unmentioned. An enumeration index is Terminating when
+ it "maps to a device that has been previously selected", so '0,0' surfaces one device
+ rather than two -- counting every token read that as two survivors and called a HIP
+ ordinal that hides the only device a valid selector. A token is Illegal when it "can't be
+ evaluated into an instance of Device UUID or Enumeration Index" (ROCR-Runtime,
+ core/inc/amd_filter_device.h), and an Illegal token ends the list, so an illegal FIRST
+ token leaves zero survivors: _post_rocr_device_count already counts it that way, while
+ this predicate called every non-index merely unresolved and the message then offered the
+ mask as something to check if the group change fails, when it is an independent blocker no
+ membership clears. A UUID is a form ROCr accepts, so it may name a device and this host
+ cannot match it against an ordinal count; calling that a blocker would invent a fault, and
+ without the arm the rule reads as "any non-index blocks"."""
+ _says(_reason_with_masks(monkeypatch, env, {"hip"}, gpu_count = 2), contains, absent)
+
+
+_REPAIR = "Repair installation"
+_JOIN_THE_GROUPS = "usermod -a -G render,video ada"
+
+
+# fmt: off
[email protected]("hip, contains, absent", _cases(
+ ("a_live_cuda_runtime_outranks_a_stale_rocm_intent", None, [_REPAIR], []),
+ ("a_live_hip_runtime_still_gets_the_node_hint_alone", "6.4.0", [_JOIN_THE_GROUPS], [_REPAIR]),
+))
+# fmt: on
+def test_an_untagged_label_is_settled_by_the_live_runtime(
+ monkeypatch, linux, hip, contains, absent
+):
+ """A conda or locally built CUDA wheel carries no +cu tag, so the label names no vendor
+ and a venv that once recorded a ROCm flavor made this read the wheel as AMD-targeted --
+ replacing the reinstall advice with group membership on a host whose CUDA build cannot
+ use the AMD card however open its nodes are. torch.version.cuda is the build's own
+ answer and is stubbed here rather than the predicate that reads it. torch.version.hip
+ set means the wheel IS a ROCm build, so the closed node explains it fully and the
+ reinstall advice would send the user after the wrong repair -- without that arm the
+ rule could be "an untagged label is never AMD"."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ _mismatch_vendors(monkeypatch, {"amd"})
+ monkeypatch.setattr(hardware, "TORCH_IMPORT_ERROR", None)
+ monkeypatch.setattr(hardware, "_expected_rocm_flavor_was_chosen", lambda: True)
+ _torch = types.SimpleNamespace(
+ version = types.SimpleNamespace(cuda = "12.8", hip = hip), __version__ = "2.11.0"
+ )
+ monkeypatch.setitem(sys.modules, "torch", _torch)
+ _says(_capability_message("torch_cuda_unavailable", "2.11.0"), contains, absent)
+
+
+_NO_BACKEND = "no GPU backend can use the AMD card"
+_THREE_NODE_PATHS = ["/dev/kfd", "/dev/dri/renderD128", "/dev/dri/renderD129"]
+
+
+# fmt: off
[email protected]("present, openable, contains, absent", _cases(
+ ("an_open_sibling_stops_it_speaking_for_the_card", _THREE_NODE_PATHS,
+ {"/dev/kfd", "/dev/dri/renderD129"},
+ ["/dev/dri/renderD128", "the card behind them", "another AMD render node"], [_NO_BACKEND]),
+ ("no_open_sibling_still_speaks_for_the_card", _AMD_NODES, {"/dev/kfd"}, [_NO_BACKEND], []),
+ ("a_kfd_only_closed_set_still_claims_only_rocm", _AMD_NODES, {"/dev/dri/renderD128"},
+ ["ROCm cannot use the AMD card"], []),
+))
+# fmt: on
+def test_how_wide_a_claim_the_closed_set_supports(
+ monkeypatch, linux, present, openable, contains, absent
+):
+ """One shut render node on a multi-AMD host leaves the other GPU fully reachable: HIP has
+ /dev/kfd plus an open render node, and the Vulkan loader has the same. Claiming "no GPU
+ backend can use the AMD card" there contradicts _explain_empty_gpu_probe, which appends
+ this sentence right after saying the closed node is NOT why the probe came back empty.
+ With the only render node shut nothing enumerates and the claim is exactly right, so the
+ fix cannot be "never claim the card", which is the #10466 message gone; and /dev/kfd with
+ an open render node beside it has no sibling of its own, so the narrowing must not weaken
+ that arm either -- Vulkan works there, ROCm does not."""
+ _nodes(monkeypatch, present = present, openable = openable)
+ _says(amd.amd_node_permission_hint(), contains, absent)
+
+
+_GID_993_PAIR = "sudo groupadd -g 993 amdgpu993 && sudo usermod -a -G amdgpu993 ada"
+_GID_994_PAIR = "sudo groupadd -g 994 amdgpu994 && sudo usermod -a -G amdgpu994 ada"
+
+
+# fmt: off
[email protected]("gids, contains, absent", _cases(
+ ("the_hint_also_adds_the_account_once_per_gid", [993, 994],
+ ["993, 994", "each of them", _GID_993_PAIR, _GID_994_PAIR, _BOTH_FLAGS], []),
+ ("a_single_gid_reads_singular", [993],
+ ["GID 993", "create a group for it", "sudo groupadd -g 993"], ["GIDs"]),
+ ("and_says_to_start_a_new_session", [993], ["groupadd -g 993 amdgpu993", "log out and back in"],
+ []),
+ ("and_is_a_command_a_shell_will_run", [993, 994], [_GID_993_PAIR, _GID_994_PAIR], ["<", ">"]),
+))
+# fmt: on
+def test_the_unnamed_gid_hint(monkeypatch, linux, gids, contains, absent):
+ """groupadd gives the numeric owner a NAME. It does not put this account in the group, so
+ a user who follows the sentence to the letter still cannot open the node: both halves are
+ needed, one pair per GID, since one groupadd names one numeric owner and the second node
+ stays shut for anyone who runs only the first command. The plural wording must not be the
+ only wording either, or one GID reads as two and stops matching what it printed. usermod
+ changes /etc/group, not the groups the running login holds, so the named-group branch has
+ always ended "then log out and back in"; the unnamed-GID branch prescribes the same
+ usermod and said nothing, so an immediate retry fails and the command reads as the one
+ that did not work. `<name>` is not a placeholder in a shell, it is a redirection:
+ `groupadd -g 993 <name>` parses as a read from ./name followed by a `>` with no target,
+ which bash, dash and sh all reject with a syntax error before groupadd runs. Every
+ character here is meant to be pasted, so it must carry no shell metacharacter it does not
+ mean. The name is derived from the GID, which by definition here has no group entry."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ monkeypatch.setattr(amd, "_has_an_access_acl", lambda path: False)
+ _owning(monkeypatch, unnamed = gids)
+ _says(amd.amd_node_permission_hint(), contains, absent)
+
+
+def _install_sh_stat_format() -> str:
+ """The stat format install.sh reads its node records in, lifted rather than restated."""
+ line = next(_l for _l in _install_sh_lines() if "stat -c '" in _l and '"$_anr_node"' in _l)
+ return line.split("'")[1]
+
+
+def _install_sh_classify(
+ *,
+ mode: str = "660",
+ group: str = "UNKNOWN",
+ gid: str = "0",
+ path: str = "/dev/kfd",
+ uid: str = "0",
+ self_uid: str = "4242",
+) -> str:
+ """One classified line from the real _amd_node_repairs, for a synthetic stat record.
+
+ The awk program is lifted whole, and only ``stat`` and ``ls`` are stubbed, because the
+ cases that matter cannot be built as files: a node whose GID has no group entry needs a
+ GID this host does not name, and a root-owned one needs root.
+
+ The record is assembled in install.sh's OWN field order, read off the shipped format
+ string, so a case names the field it means instead of counting pipes. A hand-written
+ record in a fixed order would keep passing after the format was reordered while
+ measuring the wrong fields, which is exactly what the order exists to defend against:
+ the group NAME is the one field NSS controls, so it is last, and a name carrying the
+ separator can no longer shift the mode, the GID or the uid that the classifier branches
+ on.
+ """
+ _by_spec = {"%a": mode, "%G": group, "%g": gid, "%n": path, "%u": uid}
+ stat_line = "|".join(_by_spec[_spec] for _spec in _install_sh_stat_format().split("|"))
+ lines = _install_sh_lines()
+ script = "\n".join(
+ [
+ f"stat() {{ printf '%s\\n' {shlex.quote(stat_line)}; }}",
+ "ls() { printf '%s\\n' '-rw-rw---- 1 root root 0 Jan 1 00:00 node'; }",
+ f"id() {{ echo {self_uid}; }}",
+ _shell_fn(lines, "_amd_node_repairs"),
+ "_amd_node_repairs /dev/kfd",
+ ]
+ )
+ return _install_sh_run(script).strip()
+
+
+def _install_sh_diag_route(leaf: str) -> str:
+ """Whether install.sh routes a given torch index leaf into the AMD node diagnosis.
+
+ The case statement is lifted whole rather than re-expressed, so the globs under test
+ are the shipped ones; only the leaf it reads is supplied, through a stubbed extractor,
+ because the URL-to-leaf step has its own tests.
+ """
+ lines = _install_sh_lines()
+ start = next(i for i, line in enumerate(lines) if line.startswith("_amd_node_diag_leaf="))
+ end = next(i for i in range(start, len(lines)) if lines[i] == "esac")
+ script = "\n".join(
+ [
+ _shell_fn(lines, "_is_pip_rocm_family_leaf"),
+ f"_torch_index_url_leaf() {{ printf '%s' {shlex.quote(leaf)}; }}",
+ "TORCH_INDEX_URL=stub",
+ "\n".join(lines[start : end + 1]),
+ 'printf "%s" "$_amd_node_diag_route"',
+ ]
+ )
+ return _install_sh_run(script).strip()
+
+
+def _install_sh_kfd_scope(
+ closed_nodes: str,
+ *,
+ skip_torch: bool,
+ backend: "str | None",
+ torch_index: str = "https://download.pytorch.org/whl/rocm6.4",
+ nvidia: bool = False,
+) -> str:
+ """The closed-node message with the KFD scoping in front of it.
+
+ A separate lift from _install_sh_hint because the filter sits ABOVE the block that
+ harness extracts -- deliberately, so the diagnosis itself stays one self-contained
+ block -- and the thing under test here is which nodes reach it.
+ """
+ lines = _install_sh_lines()
+ _filter_start = next(
+ i for i, line in enumerate(lines) if line == "if ! _run_may_open_kfd; then"
+ )
+ _filter_end = next(i for i in range(_filter_start, len(lines)) if lines[i] == "fi")
+ block_start = _install_sh_if(lines, '[ -n "$_closed_amd_nodes" ]; then')
+ end = next(i for i in range(block_start, len(lines)) if lines[i] == "fi")
+ script = "\n".join(
+ [
+ 'substep() { echo "$1"; }',
+ 'C_WARN=""',
+ "_amd_render_node_present() { return 0; }",
+ 'id() { case "$1" in -un) echo ada ;; *) echo 4242 ;; esac; }',
+ "_amd_node_diag_route=true",
+ "OS=linux",
+ f"SKIP_TORCH={'true' if skip_torch else 'false'}",
+ "_amd_node_repairs() { printf '%s\\n' 'join:render'; }",
+ *_run_scope_defs(lines, nvidia = nvidia),
+ _shell_fn(lines, "_run_may_open_a_gpu_node"),
+ "\n".join(lines[_filter_start : _filter_end + 1]),
+ "\n".join(lines[block_start : end + 1]),
+ ]
+ )
+ env = _install_sh_env(closed_nodes, "ada", backend, torch_index)
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, env = env)
+ assert out.returncode == 0, out.stderr
+ return out.stdout
+
+
+_CPU_INDEX = "https://download.pytorch.org/whl/cpu"
+_ROCM_INDEX = "https://download.pytorch.org/whl/rocm6.4"
+_CLOSED = "cannot open its device nodes"
+_KFD_REPORTED = [_CLOSED, "/dev/kfd"]
+
+
+# fmt: off
[email protected]("closed, skip_torch, backend, index, contains, absent", _cases(
+ ("a_vulkan_only_no_torch_run_is_not_sent_after_kfd", "/dev/kfd", True, "vulkan", _ROCM_INDEX,
+ None, []),
+ ("no_torch_alone_still_reports_a_closed_kfd", "/dev/kfd", True, None, _ROCM_INDEX,
+ _KFD_REPORTED, []),
+ ("a_vulkan_only_run_still_reports_a_closed_render_node", f"/dev/kfd\n{_RENDER}", True, "vulkan",
+ _ROCM_INDEX, [_RENDER], ["/dev/kfd"]),
+ ("a_torch_install_is_unaffected_by_the_backend_request", "/dev/kfd", False, "vulkan",
+ _ROCM_INDEX, ["/dev/kfd"], []),
+ ("a_no_torch_cpu_run_is_not_sent_after_a_closed_render_node", _RENDER, True, "cpu", _ROCM_INDEX,
+ None, []),
+ ("a_no_torch_cpu_run_is_silent_about_the_kfd_node_too", "/dev/kfd", True, "cpu", _ROCM_INDEX,
+ None, []),
+ ("a_torch_install_asking_for_cpu_llama_still_reports_its_nodes", _RENDER, False, "cpu",
+ _ROCM_INDEX, [_CLOSED], []),
+ ("a_no_torch_cuda_run_is_not_sent_after_the_kfd_node", "/dev/kfd", True, "cuda", _ROCM_INDEX,
+ None, []),
+ ("a_no_torch_cuda_run_is_not_sent_after_the_render_node_either", _RENDER, True, "cuda",
+ _ROCM_INDEX, None, []),
+ ("a_torch_install_asking_for_cuda_llama_still_reports_its_nodes", "/dev/kfd", False, "cuda",
+ _ROCM_INDEX, [_CLOSED], []),
+ ("a_backend_value_with_internal_whitespace_is_not_a_backend", "/dev/kfd", True, "vul kan",
+ _ROCM_INDEX, _KFD_REPORTED, []),
+ ("the_same_value_spelled_properly_is_still_a_backend", "/dev/kfd", True, " VULKAN ",
+ _ROCM_INDEX, None, []),
+ ("a_cpu_torch_index_with_a_vulkan_bundle_is_not_sent_after_kfd", "/dev/kfd", False, "vulkan",
+ _CPU_INDEX, None, []),
+ ("a_cpu_torch_index_alone_still_reports_a_closed_kfd", "/dev/kfd", False, None, _CPU_INDEX,
+ _KFD_REPORTED, []),
+ ("a_rocm_torch_index_is_unaffected_by_a_vulkan_bundle", "/dev/kfd", False, "vulkan",
+ _ROCM_INDEX, _KFD_REPORTED, []),
+))
+# fmt: on
+def test_which_closed_nodes_reach_the_installers_diagnosis(
+ closed, skip_torch, backend, index, contains, absent
+):
+ """/dev/kfd is opened by ROCm and nothing else, the shell twin of the runtime's needs_kfd.
+ SKIP_TORCH cannot decide it alone: --no-torch still installs a GGUF bundle, the ROCm one
+ opens /dev/kfd exactly as torch would, and which it will be is chosen later in setup.sh,
+ so suppressing there hides the #10466 diagnosis from the GGUF users it was written for.
+ Vulkan opens the render node, so the scoping takes /dev/kfd and nothing else; dropping the
+ whole diagnosis would silence the node that blocks every backend. ROCm torch opens
+ /dev/kfd whatever the bundle, so a request cannot scope the torch diagnosis.
+
+ A CPU bundle beside --no-torch opens no GPU node at all, so the group and udev repair
+ described a card nothing in the run would touch: only /dev/kfd was filtered, right for
+ Vulkan and one node short for CPU, and the narrower rule must survive the wider one.
+ REQUESTABLE_BACKENDS is auto/cpu/cuda/rocm/vulkan and a CUDA bundle opens /dev/nvidia* and
+ neither AMD node, so that run was handed the same repairs. Both report again once torch is
+ installing, which is why the predicate reads BOTH halves.
+
+ An explicitly CPU torch index installs a wheel with no ROCm runtime in it, so with a
+ non-ROCm bundle too nothing opens /dev/kfd; the early return declared otherwise purely
+ because torch was being installed at all. "cpu" is deliberately kept in the diagnosis
+ route one layer up, since the GGUF bundle is chosen later and may be the ROCm one, and a
+ ROCm index opens /dev/kfd whatever the bundle, so the rule is not "any explicit non-ROCm
+ backend suppresses".
+
+ The bundle selector normalizes with `awk '{$1=$1}'` -- trim and collapse, never delete --
+ then REJECTS what it does not recognise and falls back to automatic selection, which may
+ install ROCm. Deleting internal whitespace made "vul kan" match, so the run went quiet
+ about the very node selection needs, while a padded upper-case value does name Vulkan and
+ must still be scoped. Asserted through the KFD scope rather than the whole diagnosis,
+ because the wider predicate matches cpu|cuda only and lets every Vulkan spelling through
+ either way, which made an earlier version of this test pass on both forms."""
+ out = _install_sh_kfd_scope(closed, skip_torch = skip_torch, backend = backend, torch_index = index)
+ _says(out, contains, absent)
+
+
+# fmt: off
[email protected]("error, hip, contains, absent", _cases(
+ ("an_untagged_cuda_wheel_that_will_not_import_is_still_another_vendors", "libcudart.so.13",
+ None, [_REPAIR], []),
+ ("an_unimportable_rocm_wheel_still_gets_the_node_hint_alone", "libamdhip64.so", "6.4.0",
+ [_JOIN_THE_GROUPS], [_REPAIR]),
+))
+# fmt: on
+def test_an_unimportable_torch_is_read_from_its_markers_on_disk(
+ monkeypatch, linux, error, hip, contains, absent
+):
+ """_torch_reports_a_hip_runtime answers an import failure from torch/version.py on disk;
+ its mirror returned False there instead, so the untagged-CUDA clearing was inert on the
+ one path it exists for. A stale recorded ROCm flavor then spoke for a CUDA wheel, and the
+ closed node replaced the reinstall guidance with group membership that cannot make that
+ wheel use the card. The hip reading leads because a ROCm build records hip and may record
+ cuda besides, so ordering the two the other way round would send every AMD host whose
+ torch fails to import after a reinstall it does not need."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ _mismatch_vendors(monkeypatch, {"amd"})
+ monkeypatch.setattr(hardware, "TORCH_IMPORT_ERROR", ImportError(error))
+ monkeypatch.setattr(hardware, "_expected_rocm_flavor_was_chosen", lambda: True)
+ monkeypatch.setattr(hardware, "_installed_torch_label_on_disk", lambda: "2.11.0")
+ monkeypatch.setattr(
+ hardware,
+ "_installed_torch_markers_on_disk",
+ lambda: {"cuda": "13.0", "hip": hip, "xpu": None},
+ )
+ _says(_capability_message("torch_cuda_unavailable", "2.11.0"), contains, absent)
+
+
+# fmt: off
[email protected]("present, acl, expected", _cases(
+ ("the_sentence_names_every_path_it_lists", _AMD_NODES, _AMD_NODES,
+ "getfacl /dev/kfd /dev/dri/renderD128"),
+ ("one_path_in_one_path_out", ["/dev/kfd"], ["/dev/kfd"], "getfacl /dev/kfd before"),
+))
+# fmt: on
+def test_the_acl_sentence_runs_getfacl_on_every_node_it_lists(
+ monkeypatch, linux, present, acl, expected
+):
+ """The sentence lists every ACL-carrying node and then ran getfacl on the first one
+ alone, so a user following it read one node's grant and was left with the second blocker
+ undiagnosed. ROCm needs both nodes and their ACLs need not agree. The single-node arm is
+ there so the fix cannot have introduced a stray separator into the common case."""
+ _nodes(monkeypatch, present = present, openable = set())
+ _owning(monkeypatch, acl = acl)
+ assert expected in amd.amd_node_permission_hint()
+
+
+_CUDA_INDEX = "https://download.pytorch.org/whl/cu128"
+
+
+# fmt: off
[email protected]("index_url, kwargs, routes", _cases(
+ ("a_mirror_named_for_an_arch_is_not_a_rocm_route",
+ "https://download.pytorch.org/whl/gfx-mirror", {}, False),
+ ("a_private_rocm_build_is_not_one_either", "https://example.invalid/wheels/rocm7.2-private/",
+ {}, False),
+ ("the_real_gfx_route_is_still_read_as_one",
+ "https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0/gfx1151", {}, True),
+ ("a_no_torch_vulkan_run_still_diagnoses_its_render_nodes", _CUDA_INDEX,
+ dict(skip_torch = True, backend = "vulkan"), True),
+ ("a_no_torch_cpu_backend_run_still_does_not", _CUDA_INDEX,
+ dict(skip_torch = True, backend = "cpu"), False),
+ ("a_cuda_wheel_install_asking_for_cuda_is_off_the_route", _CUDA_INDEX, dict(backend = "cuda"),
+ False),
+ ("a_cuda_wheel_install_asking_for_cpu_is_too", _CUDA_INDEX, dict(backend = "cpu"), False),
+))
+# fmt: on
+def test_which_runs_take_the_amd_node_diagnosis_route(index_url, kwargs, routes):
+ """The gate globbed the raw URL for */rocm* and */gfx*, which matches exactly the custom
+ pins _is_pip_rocm_family_leaf exists to reject -- a mirror named for an arch, or a private
+ ROCm build -- so an install not on a published ROCm route was told to repair the AMD
+ kernel stack. Classifying the leaf reuses that rejection, and it stays narrow:
+ repo.radeon.com's arch leaf is a real ROCm route.
+
+ --no-torch installs no wheel, so the index resolved above this gate describes nothing that
+ will run; on a hybrid host it is the CUDA one, and reading it as the route silenced every
+ node diagnosis for a run whose Vulkan bundle opens the very render node they are about. A
+ CPU llama.cpp bundle opens no AMD node at all, so those runs stay off and the fix is not
+ "--no-torch always diagnoses". A run installing CUDA wheels stays off the route as long as
+ its bundle opens no AMD node either, so the fix is scoped to the run that installs nothing
+ rather than reopening the arm above; an explicit rocm or vulkan request is the one case
+ that does reopen it, and it has its own arm below the override. Fails before the fix,
+ which read the unused index."""
+ assert _diag_route(index_url, **kwargs) is routes
+
+
+# fmt: off
[email protected]("needs_kfd, contains, absent", _cases(
+ ("a_vulkan_caller_is_not_told_to_map_the_kfd_node", False, ["--device /dev/dri."],
+ ["/dev/kfd"]),
+ ("a_rocm_caller_is_still_told_to_map_both", True, ["--device /dev/kfd --device /dev/dri."], []),
+))
+# fmt: on
+def test_the_mapping_advice_names_the_nodes_the_caller_opens(
+ monkeypatch, linux, needs_kfd, contains, absent
+):
+ """The mapping advice named both nodes whatever the caller was. Vulkan never opens
+ /dev/kfd -- which is why _amd_nodes_the_runtime_lacks already excludes it under
+ needs_kfd = False -- so this handed a container another host device for nothing. HIP
+ opens both, so the fix must not narrow the advice for the caller it was written for."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = {"/dev/kfd"})
+ _says(amd.amd_node_permission_hint(needs_kfd = needs_kfd), contains, absent)
+
+
+# fmt: off
[email protected]("kwargs, contains, absent", _cases(
+ ("a_no_torch_vulkan_installer_names_only_the_render_node",
+ dict(skip_torch = True, backend = "vulkan"), ["Docker that is --device /dev/dri."],
+ ["--device /dev/kfd"]),
+ ("an_ordinary_installer_run_still_names_both", {}, ["--device /dev/kfd --device /dev/dri."],
+ []),
+))
+# fmt: on
+def test_the_installers_device_pair_follows_the_run(tmp_path, kwargs, contains, absent):
+ """The installer twin: --no-torch with an explicit Vulkan bundle opens no /dev/kfd, so the
+ device pair it prints must not name one, while a torch install opens it and the pair stays."""
+ node = tmp_path / "renderD128"
+ node.write_bytes(b"")
+ node.chmod(0o660)
+ _says(_install_sh_hint(str(node), render_present = False, **kwargs), contains, absent)
+
+
+# fmt: off
[email protected]("present, openable, vendor_readable, exists, hint_says", _cases(
+ ("a_node_whose_vendor_cannot_be_read_is_not_called_absent", _AMD_NODES, set(_AMD_NODES), False,
+ True, None),
+ ("a_host_with_no_render_node_at_all_still_says_so", ["/dev/kfd"], {"/dev/kfd"}, True, False,
+ "No AMD render node"),
+))
+# fmt: on
+def test_an_unreadable_render_node_vendor_reads_as_present(
+ monkeypatch, linux, present, openable, vendor_readable, exists, hint_says
+):
+ """A container can map /dev/dri and still mask the sysfs entry that names the vendor.
+ Reading that unknown as "no AMD render node exists" told the user to recreate the
+ container with --device /dev/dri, which that shape has already done. The fix is "unknown
+ reads as present" rather than "always present": a host whose glob finds nothing has
+ nothing unreadable either, and the sentence #10466 needs must still be printed."""
+ _nodes(monkeypatch, present = present, openable = openable, vendor_readable = vendor_readable)
+ assert amd._amd_render_node_exists() is exists
+ if hint_says is None:
+ assert amd.amd_node_permission_hint() is None
+ else:
+ assert hint_says in amd.amd_node_permission_hint()
+
+
+def test_another_vendors_open_render_node_is_seen(monkeypatch, linux):
+ """The evidence the Vulkan caller needs: a node that was read, named another vendor,
+ and opens. An unreadable one is not evidence either way and must not count."""
+ monkeypatch.setattr(amd.platform, "system", lambda: "Linux")
+ monkeypatch.setattr(
+ amd.glob, "glob", lambda pattern: ["/dev/dri/renderD128", "/dev/dri/renderD129"]
+ )
+ monkeypatch.setattr(
+ amd,
+ "_render_node_vendor",
+ lambda p: "0x1002" if p.endswith("128") else "0x10de",
+ )
+ monkeypatch.setattr(amd.os, "access", lambda p, mode: p.endswith("129"))
+ assert amd.a_non_amd_render_node_is_open() is True
+ monkeypatch.setattr(amd, "_render_node_vendor", lambda p: None)
+ assert amd.a_non_amd_render_node_is_open() is False
+
+
+_PROBE_SENTENCE = "the Vulkan probe reported no device"
+
+
+def _finding(
+ reason: str,
+ *,
+ credited,
+ contains = (),
+ absent = (),
+):
+ """Check an empty-probe reason.
+
+ ``credited`` is whether another vendor's open node is still a complete path, which keeps
+ the Vulkan probe sentence in front of the reason; False means the node hint replaced it,
+ and None means this build never asks the question.
+ """
+ if credited is True:
+ assert reason.startswith(_PROBE_SENTENCE), reason
+ elif credited is False:
+ assert _PROBE_SENTENCE not in reason, reason
+ _says(reason, contains, absent)
+
+
+# fmt: off
[email protected]("openable, other_open, backends, credited, contains, absent", _cases(
+ ("a_vulkan_probe_keeps_its_finding_when_another_vendor_is_open", set(), True, {"vulkan"}, True,
+ ["Separately", "usermod"], []),
+ ("no_other_vendor_still_gives_the_hint_alone", set(), False, {"vulkan"}, False, ["usermod"],
+ []),
+ ("a_rocm_build_is_unaffected_by_another_vendors_node", {_RENDER}, True, {"hip"}, None,
+ [_JOIN_THE_GROUPS], ["Separately"]),
+))
+# fmt: on
+def test_whether_another_vendors_open_node_is_a_path_for_this_build(
+ monkeypatch, linux, openable, other_open, backends, credited, contains, absent
+):
+ """A Vulkan-only build enumerates any vendor, so an open Intel or NVIDIA render node is
+ a complete path for it: the closed AMD node cannot then be the whole reason the probe
+ came back empty, and returning it alone dropped the finding that was. With no open node
+ of any vendor the closed AMD one IS the reason, and the hint must still replace the bare
+ probe sentence. HIP needs /dev/kfd and an AMD render node, which no other vendor's node
+ substitutes for, so the question is not even asked for it."""
+ _nodes(monkeypatch, present = _AMD_NODES, openable = openable)
+ monkeypatch.setattr(amd, "a_non_amd_render_node_is_open", lambda: other_open)
+ _ggml(monkeypatch, backends)
+ _finding(_empty_probe(), credited = credited, contains = contains, absent = absent)
+
+
+def test_the_repair_names_the_account_the_access_tests_answered_for(monkeypatch, linux):
+ """USER is inherited, so a container that changes its numeric user without resetting it
+ names somebody else and the command modifies the wrong account, leaving the running one
+ still unable to open the node."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ monkeypatch.setenv("USER", "root")
+ hint = amd.amd_node_permission_hint()
+ assert hint.rstrip().endswith("render,video ada") or "render,video ada" in hint
+ assert "render,video root" not in hint
+
+
+def test_the_installer_names_the_account_id_reports(tmp_path):
+ """The shell twin: `id -un` is the account the mode tests above answered for."""
+ node, _group = _a_node_a_membership_would_open(tmp_path)
+ out = _install_sh_hint(str(node), env_user = "root", id_user = "ada")
+ # The group is whatever owns a tmp_path file on the runner, so the account is what is
+ # asserted -- naming a group here would be asserting about the runner.
+ assert re.search(r"usermod -a -G \S+ ada", out)
+ assert " root" not in out
+
+
+def test_the_installer_unnamed_gid_repair_is_runnable_too(tmp_path):
+ """The shell twin, checked the same way and then actually parsed: `bash -n` on the two
+ emitted lines is the assertion that a placeholder would fail. Without the parse this
+ would only be testing that a string changed."""
+ out = _install_sh_hint("/dev/dri/renderD128", repairs = "gid:993")
+ # Reassembled across the line continuation before being parsed. The installer prints the
+ # repair as `groupadd ... && \` then an indented `usermod ...`, and taking the lines
+ # separately parses a FRAGMENT: the first half ends in a dangling backslash and is not a
+ # command at all. GNU bash accepts one at end of input and returns 0, while bash 3.2, the
+ # one macOS ships, calls it "syntax error: unexpected end of file" -- so this passed here
+ # and failed there while testing the real command on neither.
+ _cmds: "list[str]" = []
+ _pending = ""
+ for _line in out.splitlines():
+ _stripped = _line.strip()
+ if not _pending and not _stripped.startswith(("sudo group", "sudo usermod")):
+ continue
+ _pending = f"{_pending} {_stripped}" if _pending else _stripped
+ if _pending.endswith("\\"):
+ _pending = _pending[:-1].rstrip()
+ continue
+ _cmds.append(_pending)
+ _pending = ""
+ assert not _pending, f"repair ends mid-continuation: {_pending!r}"
+ assert _cmds, out
+ for _cmd in _cmds:
+ assert "<" not in _cmd and ">" not in _cmd
+ _parsed = subprocess.run(["bash", "-n", "-c", _cmd], capture_output = True, text = True)
+ assert _parsed.returncode == 0, f"{_cmd!r}: {_parsed.stderr}"
+
+
+# fmt: off
[email protected]("vendor, verdict", _cases(
+ ("a_vendor_the_installer_cannot_read_is_not_absent", None, "PRESENT"),
+ ("a_vendor_that_names_another_is_still_absent", "0x10de\n", "ABSENT"),
+))
+# fmt: on
+def test_what_the_installer_makes_of_a_render_node_vendor(tmp_path, vendor, verdict):
+ """The shell twin of the Python rule, which round sixteen fixed on one side only. A
+ container mapping /dev/dri while denying its sysfs attributes has a render node; calling
+ that absence told the user to recreate the container with the device it already has. The
+ rule is "unknown", not "any node": a readable vendor that is not AMD is a real answer, and
+ treating it as presence would claim an AMD card on an NVIDIA-only host. Only the two path
+ ROOTS are substituted, so the logic under test is the shipped one, and a sysfs directory
+ with no vendor file in it is what a container denying the attribute looks like from here.
+ """
+ lines = _install_sh_lines()
+ fn = _shell_fn(lines, "_amd_render_node_present")
+ fn = fn.replace("/dev/dri/renderD*", f"{tmp_path}/dev/dri/renderD*")
+ fn = fn.replace("/sys/class/drm/", f"{tmp_path}/sys/class/drm/")
+ (tmp_path / "dev/dri").mkdir(parents = True)
+ (tmp_path / "dev/dri/renderD128").write_bytes(b"")
+ (tmp_path / "sys/class/drm/renderD128/device").mkdir(parents = True)
+ if vendor is not None:
+ (tmp_path / "sys/class/drm/renderD128/device/vendor").write_text(vendor)
+ out = subprocess.run(
+ [
+ "bash",
+ "-c",
+ fn + "\nif _amd_render_node_present; then echo PRESENT; else echo ABSENT; fi",
+ ],
+ capture_output = True,
+ text = True,
+ )
+ assert out.returncode == 0, out.stderr
+ assert out.stdout.strip() == verdict
+
+
+_HIP = "HIP_VISIBLE_DEVICES"
+_SHUT = dict(present = _THREE_NODE_PATHS, openable = {"/dev/kfd", "/dev/dri/renderD129"})
+_SHUT_NO_KFD = dict(present = [_RENDER, "/dev/dri/renderD129"], openable = {"/dev/dri/renderD129"})
+
+
+# fmt: off
[email protected]("host, env, needs_kfd, blocks", _cases(
+ ("a_mask_selecting_the_closed_one_leaves_no_way_in", _SHUT, {_HIP: "0"}, True, True),
+ ("no_mask_still_credits_the_open_sibling", _SHUT, {}, True, False),
+ ("an_empty_mask_is_not_a_mask", _SHUT, {_HIP: " "}, True, False),
+ ("gpu_device_ordinal_is_a_selector_too", _SHUT, {"GPU_DEVICE_ORDINAL": "1"}, True, True),
+ ("a_vulkan_probe_ignores_a_hip_mask", _SHUT_NO_KFD, {_HIP: "0"}, False, False),
+ ("a_hip_caller_on_the_same_masked_host_still_blocks", _SHUT, {_HIP: "0"}, True, True),
+))
+# fmt: on
+def test_whether_an_open_sibling_is_a_way_in(monkeypatch, linux, host, env, needs_kfd, blocks):
+ """HIP_VISIBLE_DEVICES=0 narrows the runtime to one GPU, and nothing here maps a render node
+ back to the index it was selected by, so an OPEN sibling may belong to the GPU the mask
+ excludes. Reading it as an alternative suppressed the repair for the node the run will
+ actually use and sent the user to reinstall a runtime instead. With no selector the runtime
+ is free to use the open node, so the closed one is a second finding rather than the cause --
+ the rule cannot be "a closed node always blocks", which the open sibling arm was added to
+ stop -- and an exported but empty variable narrows nothing, so reading it as a selector
+ would keep the repair on every host that merely exports the name.
+
+ GPU_DEVICE_ORDINAL is ROCm's fourth visibility variable, already modelled in
+ test_amd_smi_inventory_matches_hip.py and llama_cpp.py's own selector check; omitting it
+ left one of the four narrowing the runtime while the sibling was still credited. HIP's
+ selectors are HIP's, though: Vulkan reads none of them -- which is what needs_kfd is for --
+ so a Vulkan caller on a masked host may still use the open sibling, and returning the
+ permission hint as the sole cause would send an empty Vulkan probe after a group change that
+ cannot fix it. The HIP caller on that host must still block, or the rule becomes "ignore
+ selectors"; its host keeps /dev/kfd present AND open, or the answer comes back True on the
+ missing-node branch and says nothing about the mask, which is how that arm first passed."""
+ _nodes(monkeypatch, **host)
+ for _name, _value in env.items():
+ monkeypatch.setenv(_name, _value)
+ # _no_inherited_gpu_mask has already cleared every mask variable a case does not set.
+ assert amd.amd_closed_nodes_block_the_runtime(needs_kfd = needs_kfd) is blocks
+
+
+# fmt: off
[email protected]("gid, names, runner_uid, expected", _cases(
+ ("a_docker_owned_node_is_not_answered_with_usermod", 999, {999: "docker"}, None,
+ _buckets(privileged = ["docker"])),
+ ("an_unnamed_gid_zero_is_the_root_group_not_a_group_to_create", 0, {}, None,
+ _buckets(privileged = ["root"])),
+ ("an_unnamed_ordinary_gid_is_still_a_group_to_create", 993, {}, None,
+ _buckets(unnamed = [993])),
+ ("the_derivation_survives_a_root_test_runner", 44, {44: "render"}, 0,
+ _buckets(joinable = ["render"])),
+))
+# fmt: on
+def test_which_bucket_the_owning_group_lands_in(
+ monkeypatch, linux, gid, names, runner_uid, expected
+):
+ """Membership in docker is root by another route -- a container started with the host
+ filesystem mounted -- so prescribing it to open a GPU node is a privilege escalation dressed
+ as a device repair, exactly as for wheel. A minimal container can own the node root:root and
+ carry no group database entry for gid 0; the name lookup raises there, and filing that as an
+ ordinary unnamed GID produced `sudo groupadd -g 0 amdgpu0` plus a usermod into the ROOT
+ group, a grant far beyond the GPU and exactly what the privileged branch below the lookup
+ exists to refuse. gid 0 is the root group whether or not the database can name it, and the
+ rule is keyed on gid 0 rather than on the lookup failing because an unnamed NON-privileged
+ GID is the documented container case and must still get its groupadd pair.
+
+ CI commonly runs as root, and the fakes in this file give their nodes the root owner a real
+ device node has. POSIX resolves the owner class exclusively once the uid matches, so a
+ harness that let the two coincide would answer every group test with "this account owns it"
+ and assert nothing about the classification under test. The last case fails before that fix
+ with os.getuid patched to 0, which is what a root runner is."""
+ if runner_uid is not None:
+ monkeypatch.setattr(amd.os, "getuid", lambda: runner_uid)
+ _stat_nodes(monkeypatch, {"/dev/kfd": (gid, 0o660, 0)}, names)
+ assert amd._groups_that_own(["/dev/kfd"]) == expected
+
+
+# fmt: off
[email protected]("name, expected", _cases(
+ # usermod -G takes a COMMA-SEPARATED list, so this is two groups to it and the
+ # account lands in sudo. _PRIVILEGED_GROUPS compares whole names and never matches.
+ ("a_name_carrying_the_usermod_separator", "render,sudo", _buckets(unnamed = [993])),
+ # The shell twin parses `stat -c` output with awk -F'|', where such a name shifts
+ # every field after it. Refused in both halves for one rule rather than two.
+ ("a_name_carrying_the_field_separator", "render|x", _buckets(unnamed = [993])),
+ ("a_name_carrying_a_shell_metacharacter", "render;id", _buckets(unnamed = [993])),
+ ("a_name_that_is_only_whitespace", " ", _buckets(unnamed = [993])),
+ # The controls, without which this collapses into "never prescribe a group" and
+ # deletes what #10466 asked for. A Samba machine account carries the trailing $,
+ # and a distribution group may carry a dot or a dash.
+ ("an_ordinary_group_name", "render", _buckets(joinable = ["render"])),
+ ("a_samba_machine_account", "host$", _buckets(joinable = ["host$"])),
+ ("a_dotted_distribution_group", "gpu.users-1", _buckets(joinable = ["gpu.users-1"])),
+))
+# fmt: on
+def test_a_group_name_usermod_would_read_as_structure_is_not_prescribed(
+ monkeypatch, name, expected
+):
+ """Shell quoting is the wrong layer for this, which is why it was not caught by it.
+
+ `_shell_word` hands the shell ONE argument, correctly; the splitting that grants sudo
+ happens inside usermod afterwards, on its own comma syntax. So a name outside the
+ portable groupadd(8) charset is treated as no name at all and the node is reported by
+ its GID, which is what `--group-add` takes anyway, so nothing is lost.
+
+ Reverting the check puts 'render,sudo' back in the joinable bucket and the printed
+ command back to `usermod -a -G render,sudo`."""
+ _stat_nodes(monkeypatch, {"/dev/kfd": (993, 0o660, 0)}, {993: name})
+ assert amd._groups_that_own(["/dev/kfd"]) == expected
+
+
+def test_the_printed_command_never_names_a_group_it_did_not_mean(monkeypatch, linux):
+ """End to end through the message, since the bucket above is only half the story: what
+ matters is what the user pastes. The GID repair is what this host gets instead."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ _stat_nodes(monkeypatch, {"/dev/kfd": (993, 0o660, 0)}, {993: "render,sudo"})
+ _the_real_group_derivation(monkeypatch)
+ hint = amd.amd_node_permission_hint()
+ # The name is never handed to usermod, in any form. Not a bare "sudo not in hint": the
+ # GID repair legitimately says `sudo groupadd`, and an assertion that cannot tell the
+ # two apart would have to be weakened later rather than tightened.
+ assert "render,sudo" not in hint
+ assert "-G render" not in hint
+ # What this host gets instead: the GID, which is what --group-add takes anyway.
+ assert "--group-add 993" in hint
+
+
+# fmt: off
[email protected]("record, classified", _cases(
+ ("the_installer_also_refuses_an_unnamed_gid_zero", {"group": "UNKNOWN", "gid": "0"},
+ "privileged:root"),
+ ("and_still_names_an_ordinary_unnamed_gid", {"group": "UNKNOWN", "gid": "993"}, "gid:993"),
+ ("and_keeps_the_name_of_a_named_root_group", {"group": "wheel", "gid": "0"},
+ "privileged:wheel"),
+ # usermod -G takes a comma-separated list, so a group genuinely named "render,sudo"
+ # is TWO groups to it and the alternation above, which compares whole names, walks
+ # straight past it. Quoting is the wrong layer: the split happens inside usermod,
+ # after the shell has handed it one argument. Reported by GID instead.
+ ("a_name_carrying_the_usermod_separator_is_not_prescribed",
+ {"group": "render,sudo", "gid": "993"}, "gid:993"),
+ # And the field-shift the format order defends against: with the name anywhere but
+ # last, a pipe inside it moved the GID and uid the classifier branches on.
+ ("a_name_carrying_the_field_separator_is_not_prescribed", {"group": "render|x", "gid": "993"},
+ "gid:993"),
+ # The control for both, so this cannot collapse into "never prescribe a group".
+ ("an_ordinary_group_name_is_still_prescribed", {"group": "render", "gid": "993"},
+ "join:render"),
+))
+# fmt: on
+def test_how_the_installer_classifies_a_nodes_owner(record, classified):
+ """The shell twin, run through the real classifier rather than a stubbed repair line.
+ `stat -c %G` prints UNKNOWN for a gid the group database cannot name, and the unnamed
+ test ran first, so a root-owned node in a minimal container came back as `gid:0` and the
+ installer printed groupadd -g 0 plus a usermod into the root group. The same shape one
+ GID up is the documented container case and must still come back as a GID to create a
+ group for, and gid 0 WITH an entry keeps the name it has, so the reordering did not turn
+ every privileged node into the literal word root."""
+ assert _install_sh_classify(**record) == classified
+
+
+# fmt: off
[email protected]("leaf, routes", [
+ ("rocm-rel-7.0-private", False), ("rocm-rel-7.0.beta", False), ("rocm-rel-6.1", True),
+ ("rocm-rel-6.4", True), ("rocm-rel-6.5.0", True), ("rocm-rel-7.0", True),
+ ("rocm-rel-7.2.1", True), ("rocm-rel-7.3.1", True), ("gfx110X-all", True),
+ ("gfx120X-all", True), ("gfx103X-all", True), ("gfx1151", True),
+])
+# fmt: on
+def test_which_index_leaves_the_installer_reads_as_a_rocm_route(leaf, routes):
+ """repo.radeon.com publishes rocm-rel-6.4, rocm-rel-6.5.0, rocm-rel-7.2.1 -- digits and
+ dots after the prefix, nothing else. A pin that merely starts with it is a mirror, and it
+ may serve wheels that open no AMD node, so it gets the same anchoring the sibling
+ rocm[0-9]* arm already applies to rocm7.2-private. The anchor is on the character class
+ rather than a fixed version shape, since all six rocm-rel leaves this repository names
+ must keep routing, two and three components alike. The gfx family was deliberately left
+ alone: AMD's own indexes are gfx110X-all, gfx120X-all and gfx103X-all, so a suffix there
+ is the naming convention, and anchoring gfx the same way would drop routes we ship."""
+ assert _install_sh_diag_route(leaf) == ("true" if routes else "false"), leaf
+
+
+def test_the_passwd_stub_answers_a_positional_read(monkeypatch):
+ """getpass.getuser() reads pwd.getpwuid(os.getuid())[0] once none of LOGNAME, USER,
+ LNAME or USERNAME is set, and pytest calls it while building tmp_path. A stub that only
+ carries pw_name raises TypeError there, which pytest does not catch, so the whole suite
+ would die in fixture setup on a runner with no username in its environment rather than
+ run. Reproduced by clearing all four, which is the only thing that makes it reachable."""
+ for _var in ("LOGNAME", "USER", "LNAME", "USERNAME"):
+ monkeypatch.delenv(_var, raising = False)
+ assert getpass.getuser() == "ada"
+
+
+_ALSO_IN_FORCE = "visibility mask is also in force"
+_CUDA = "CUDA_VISIBLE_DEVICES"
+
+
+# fmt: off
[email protected]("env, contains, absent", _cases(
+ ("an_unusable_hip_selector_is_a_blocker", {_HIP: "garbage"},
+ ["HIP_VISIBLE_DEVICES='garbage'", _ALSO_IN_FORCE], []),
+ ("an_empty_hip_mask_defers_to_the_cuda_one_below_it", {_HIP: "", _CUDA: "0"},
+ [_JOIN_THE_GROUPS], ["visibility mask"]),
+ ("the_cuda_mask_under_it_still_blocks_when_it_hides", {_HIP: "", _CUDA: "-1"},
+ ["CUDA_VISIBLE_DEVICES='-1'", _ALSO_IN_FORCE], []),
+))
+# fmt: on
+def test_how_the_hip_selector_chain_is_read(monkeypatch, linux, env, contains, absent):
+ """clr's list terminates at the first token it cannot use, and a token does not have to
+ look numeric to be unusable: rocdevice.cpp takes `index = atoi(str_id)` and rejects it
+ unless `str_id` is that index written back out, so HIP_VISIBLE_DEVICES=garbage leaves zero
+ agents exactly as -1 does and joining the group leaves the probe as empty as it was. That
+ arm fails before the fix, which asked only whether the first token was a digit and let
+ everything else through as "not a filter".
+
+ The precedence test in clr is on the first BYTE of the value, not on whether the variable
+ exists: its flag defaults to the empty string, so an empty HIP mask reads exactly like an
+ unset one and the CUDA value below it selects devices. That value names a device in the
+ second case, so nothing is hidden and no mask is a second blocker; before the fix an empty
+ HIP mask won the chain and was reported as hiding every device. Deferring to CUDA is only
+ right if CUDA is then judged on its own merits, so a CUDA value that names no device must
+ still be reported, or the fix becomes "an empty HIP mask silences the whole chain"."""
+ _says(_reason_with_masks(monkeypatch, env, {"hip"}), contains, absent)
+
+
+_KFD_GPU_NODE = "vendor_id 4098\nsimd_count 8\n"
+_KFD_CPU_NODE = "vendor_id 0\nsimd_count 0\n"
+
+
+def _kfd_topology(monkeypatch, entries: dict):
+ """Stub /sys/class/kfd: entry name -> its properties text, or None for unreadable."""
+ monkeypatch.setattr(amd.os, "listdir", lambda _path: list(entries))
+
+ def _open(path, *_a, **_k):
+ _text = entries.get(os.path.basename(os.path.dirname(str(path))))
+ if _text is None:
+ raise OSError("unreadable")
+ return io.StringIO(_text)
+
+ monkeypatch.setattr(amd, "open", _open, raising = False)
+
+
+# fmt: off
[email protected]("entries, count", _cases(
+ ("the_cpu_node_every_topology_carries_is_excluded", {"0": _KFD_CPU_NODE, "1": _KFD_GPU_NODE},
+ 1),
+ ("an_unreadable_entry_makes_the_whole_count_unknown",
+ {"0": _KFD_GPU_NODE, "1": None, "2": _KFD_GPU_NODE}, None),
+))
+# fmt: on
+def test_the_gpu_count_reads_the_topology(monkeypatch, entries, count):
+ """The control for the helper itself: the CPU node every KFD topology carries is excluded
+ and the GPU node is counted. One entry temporarily unreadable on a two-GPU host used to
+ answer 1 rather than unknown, and an understated bound is what calls a valid selector a
+ blocker: with a count of 1, HIP_VISIBLE_DEVICES=1 reads as hiding every device and the user
+ is told to clear a mask that hides nothing. Unknown bounds nothing, which is the documented
+ contract and what _amd_render_node_exists already does with an unreadable vendor."""
+ _kfd_topology(monkeypatch, entries)
+ assert amd.amd_kfd_gpu_node_count() == count
+
+
+# fmt: off
[email protected]("render_open, contains, absent", _cases(
+ # The repair is unchanged: these nodes are still shut and membership opens them.
+ ("the_claim_is_scoped_when_a_sibling_node_is_open", True,
+ ["another AMD", "render node on this host is open", "usermod -a -G"], []),
+ ("and_covers_every_backend_when_none_is", False,
+ ["Every backend needs them, ROCm and Vulkan alike."], ["render node on this host is open"]),
+))
+# fmt: on
+def test_how_wide_a_claim_the_installer_makes(tmp_path, render_open, contains, absent):
+ """The Python half already says an open sibling means the closed nodes stop the card
+ rather than the host, and the installer claimed every backend was blocked regardless --
+ on a host where a Vulkan run is working through the open node as the user reads it. With
+ no AMD node open anywhere the closed set does block every backend, so the fix must not
+ soften the claim into always saying maybe.
+
+ Fails before the fix, which printed the unconditional claim."""
+ node, _group = _a_node_a_membership_would_open(tmp_path)
+ _says(_install_sh_hint(str(node), render_open = render_open), contains, absent)
+
+
+_AMD_MANIFEST = "@amd-manifest"
+_AMD_MANIFEST_WITHOUT_ITS_LIBRARY = "@amd-manifest-without-its-library"
+
+
+def _icd_list_value(tmp_path, value):
+ """Resolve a parametrized driver-list value, writing a manifest for the two sentinels."""
+ if value == _AMD_MANIFEST:
+ return _icd_manifest(tmp_path, "radeon_icd.json")
+ if value == _AMD_MANIFEST_WITHOUT_ITS_LIBRARY:
+ return _icd_manifest(tmp_path, "radeon_icd.json", present = False)
+ return value
+
+
+# fmt: off
[email protected]("var, value, credited, contains", _cases(
+ ("a_replacing_list_naming_amd_alone_stops_the_other_vendor_excusing_it", "VK_DRIVER_FILES",
+ _AMD_MANIFEST, False, ["usermod"]),
+ ("the_deprecated_spelling_of_that_override_counts_too", "VK_ICD_FILENAMES", _AMD_MANIFEST,
+ False, []),
+ ("the_additive_variable_leaves_the_other_vendor_credited", "VK_ADD_DRIVER_FILES",
+ "/opt/extra/icd.json", True, ["Separately"]),
+ ("an_empty_override_is_not_an_override", "VK_DRIVER_FILES", "", True, []),
+))
+# fmt: on
+def test_which_loader_variable_replaces_the_driver_search(
+ monkeypatch, linux, tmp_path, var, value, credited, contains
+):
+ """VK_DRIVER_FILES REPLACES the loader's driver search rather than adding to it, and the
+ probe child inherits it, so a list naming AMD alone means the loader never opened the
+ other vendor's driver; crediting its node suppressed the closed-node hint on a run that
+ had no other path, which hides a real repair. VK_ICD_FILENAMES is the deprecated name for
+ the same replacing list, honoured when VK_DRIVER_FILES is unset. VK_ADD_DRIVER_FILES ADDS
+ to the standard search, so every driver the loader would have found is still found and the
+ other vendor's open node is still a complete path: without that arm the fix could be "any
+ VK_ variable suppresses", turning the finding off for a host that set the one variable
+ that changes nothing about which vendors load. The loader treats an empty value as unset,
+ so a blank VK_DRIVER_FILES must not suppress either. Fails before the fix, which asked
+ only whether the node was open."""
+ reason = _vulkan_reason_under_icd_list(monkeypatch, _icd_list_value(tmp_path, value), var = var)
+ _finding(reason, credited = credited, contains = contains)
+
+
+_BOTH_VENDORS = os.pathsep.join(
+ ["/etc/vulkan/icd.d/radeon_icd.x86_64.json", "/etc/vulkan/icd.d/nvidia_icd.json"]
+)
+
+
+# fmt: off
[email protected]("value, filters, credited, contains", _cases(
+ ("a_list_naming_another_vendor_does_not_suppress_the_vulkan_finding",
+ "/etc/vulkan/icd.d/intel_icd.x86_64.json", None, True, ["Separately", "usermod"]),
+ ("a_list_carrying_both_vendors_does_not_suppress_either", _BOTH_VENDORS, None, True, []),
+ ("an_unclassifiable_entry_keeps_the_unpinned_behaviour", "/opt/vendor/icd.d", None, True, []),
+ ("an_amd_manifest_whose_library_is_gone_is_not_a_driver", _AMD_MANIFEST_WITHOUT_ITS_LIBRARY,
+ None, True, []),
+ ("an_amd_manifest_that_is_not_there_at_all_is_not_a_driver",
+ "/nonexistent/icd.d/radeon_icd.x86_64.json", None, True, []),
+ ("an_amd_driver_the_loader_filters_out_is_not_a_driver", _AMD_MANIFEST,
+ {"VK_LOADER_DRIVERS_DISABLE": "radeon*"}, True, []),
+ ("a_valid_amd_manifest_still_suppresses", _AMD_MANIFEST, {}, False, ["usermod"]),
+))
+# fmt: on
+def test_which_driver_lists_are_evidence_that_amd_is_all_the_loader_has(
+ monkeypatch, linux, tmp_path, value, filters, credited, contains
+):
+ """A list pinned to an Intel or NVIDIA ICD leaves the loader unable to use the AMD card at
+ all, so its closed node cannot be why the probe was empty, and the group repair alone
+ sends the user after a change that cannot help. Only an AMD-only list is evidence the
+ other vendor is unreachable, and a list is AMD-only or it is not: one non-AMD entry leaves
+ that vendor loadable and its open node a complete path. A directory, or a name this does
+ not recognise, is evidence of nothing, and the guard fires on positive evidence alone, so
+ an unreadable list leaves the finding as it is rather than suppressing on a guess.
+
+ A registration with no library behind it loads nothing, so a list holding only that leaves
+ the loader with no driver at all -- the filename says AMD; the manifest says nothing is
+ there -- as does a manifest missing outright, the shape a stale VK_DRIVER_FILES has after
+ a driver is uninstalled. VK_LOADER_DRIVERS_DISABLE applies to a forced list too, so naming
+ AMD and then disabling it leaves the loader with nothing; that manifest is entirely valid
+ and only the filter makes it unloadable, which separates it from the two arms above. The
+ last case is the control for all of them: present, parses, points at a library that exists
+ and survives the filters IS an AMD-only list, and the other vendor's node is then
+ unreachable. Without it the fix could be "never suppress". Fails before the fix, which
+ read any non-empty list as AMD's, and the name alone as a driver."""
+ reason = _vulkan_reason_under_icd_list(
+ monkeypatch, _icd_list_value(tmp_path, value), filters = filters
+ )
+ _finding(reason, credited = credited, contains = contains)
+
+
+def test_every_amd_icd_spelling_the_installer_knows_counts_here_too():
+ """The two lists are the same convention twice, and a driver named in one but not the
+ other would make the installer and this diagnosis disagree about the same host."""
+ import install_llama_prebuilt
+ assert amd._AMD_VULKAN_ICD_NEEDLES == install_llama_prebuilt._AMD_VULKAN_ICD_NEEDLES
+
+
+def test_a_blocking_hip_mask_cancels_the_verdict_before_any_node_advice(monkeypatch):
+ """A mask that hides every accelerator is the configuration working, so the whole
+ chat-only classification is cancelled and no message -- node repair included -- is
+ built. This is the first of the two gates that keep a blocking selector away from the
+ node hint, and it sits well above the code that returns it."""
+ monkeypatch.setattr(
+ hardware,
+ "get_physical_gpu_inventory",
+ lambda **_kw: {"devices": [{"vendor": "amd"}], "unknown": False},
+ )
+ for _var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(_var, raising = False)
+ assert hardware._masks_hide_every_accelerator(block_inventory = True) is False
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "-1")
+ assert hardware._masks_hide_every_accelerator(block_inventory = True) is True
+ assert hardware.classify_torch_build(block_inventory = True) is None
+
+
+def test_a_blocking_mask_drops_amd_from_the_vendors_the_node_hint_needs(monkeypatch):
+ """The second gate, for the hybrid host the first one does not cover: an NVIDIA card still
+ raises the verdict, but the masked AMD card is dropped from the inventory that establishes
+ it, so "amd" never reaches CHAT_ONLY_MISMATCH_VENDORS and the node hint is not even
+ computed. An empty active CUDA mask is the same story, since HIP reads that variable too."""
+ devices = [{"vendor": "amd"}, {"vendor": "nvidia"}]
+ monkeypatch.setattr(
+ hardware,
+ "get_physical_gpu_inventory",
+ lambda **_kw: {"devices": devices, "unknown": False},
+ )
+ monkeypatch.setattr(hardware, "_expected_rocm_flavor_was_chosen", lambda: True)
+ for _var in ("HIP_VISIBLE_DEVICES", "ROCR_VISIBLE_DEVICES", "CUDA_VISIBLE_DEVICES"):
+ monkeypatch.delenv(_var, raising = False)
+ kept = hardware._devices_that_can_establish_a_mismatch(devices)
+ assert {device["vendor"] for device in kept} == {"amd", "nvidia"}
+ monkeypatch.setenv("HIP_VISIBLE_DEVICES", "-1")
+ kept = hardware._devices_that_can_establish_a_mismatch(devices)
+ assert {device["vendor"] for device in kept} == {"nvidia"}
+ # HIP reads CUDA_VISIBLE_DEVICES too, so an emptied one hides both cards rather than
+ # just the NVIDIA half -- nothing establishes the mismatch at all and the verdict is
+ # cancelled a step earlier. Either way "amd" is not among the vendors, which is the
+ # invariant the node hint is gated on.
+ monkeypatch.delenv("HIP_VISIBLE_DEVICES", raising = False)
+ monkeypatch.setenv("CUDA_VISIBLE_DEVICES", "")
+ kept = hardware._devices_that_can_establish_a_mismatch(devices)
+ assert {device["vendor"] for device in kept} == set()
+
+
+def _icd_manifest(
+ tmp_path,
+ name,
+ *,
+ library = "libvulkan_radeon.so",
+ present = True,
+):
+ """A Vulkan ICD manifest on disk, and the path to it.
+
+ Written rather than named, because the rule under test is that a manifest has to point
+ at a library that is actually there: a path string alone proves nothing.
+ """
+ lib = tmp_path / library
+ if present:
+ lib.write_bytes(b"")
+ path = tmp_path / name
+ path.write_text(
+ json.dumps(
+ {
+ "file_format_version": "1.0.0",
+ "ICD": {"library_path": str(lib), "api_version": "1.3.0"},
+ }
+ ),
+ encoding = "utf-8",
+ )
+ return str(path)
+
+
+def _vulkan_reason_under_icd_list(
+ monkeypatch,
+ value,
+ *,
+ var = "VK_DRIVER_FILES",
+ search_dirs = None,
+ filters = None,
+):
+ """The empty-probe reason for a Vulkan build with the AMD node shut and another
+ vendor's node open, under a given loader configuration.
+
+ ``value`` is a forced driver list, or None for a host that has none and is answered by
+ the loader's own search; ``search_dirs`` stands in for that search, so an arm cannot be
+ decided by whatever drivers the runner happens to have installed."""
+ _nodes(monkeypatch, present = _AMD_NODES, openable = set())
+ monkeypatch.setattr(amd, "a_non_amd_render_node_is_open", lambda: True)
+ for _var in ("VK_DRIVER_FILES", "VK_ICD_FILENAMES", "VK_ADD_DRIVER_FILES"):
+ monkeypatch.delenv(_var, raising = False)
+ if filters is not None or search_dirs is not None:
+ # Cleared only for the arms that state their own loader configuration, since an arm
+ # that names a filter is testing that filter and must keep it.
+ for _var in ("VK_LOADER_DRIVERS_SELECT", "VK_LOADER_DRIVERS_DISABLE"):
+ monkeypatch.delenv(_var, raising = False)
+ for _var, _value in (filters or {}).items():
+ monkeypatch.setenv(_var, _value)
+ if search_dirs is not None:
+ monkeypatch.setattr(amd, "_vulkan_icd_search_dirs", lambda: list(search_dirs))
+ if value is not None:
+ monkeypatch.setenv(var, value)
+ _ggml(monkeypatch, {"vulkan"})
+ return _empty_probe()
+
+
+def test_the_loader_filter_rule_matches_the_installers(tmp_path):
+ """Both halves implement the loader's four globs, and a host where they disagree gets
+ one answer from the Vulkan route and another from this diagnosis."""
+ import install_llama_prebuilt
+
+ cases = [
+ ("radeon_icd.x86_64.json", "radeon*"),
+ ("radeon_icd.x86_64.json", "*radeon*"),
+ ("radeon_icd.x86_64.json", "*json"),
+ ("radeon_icd.x86_64.json", "radeon_icd.x86_64.json"),
+ ("radeon_icd.x86_64.json", "nvidia*"),
+ ("nvidia_icd.json", "radeon*"),
+ ]
+ for name, pattern in cases:
+ for var in ("VK_LOADER_DRIVERS_DISABLE", "VK_LOADER_DRIVERS_SELECT"):
+ os.environ.pop("VK_LOADER_DRIVERS_DISABLE", None)
+ os.environ.pop("VK_LOADER_DRIVERS_SELECT", None)
+ os.environ[var] = pattern
+ try:
+ assert amd._vulkan_loader_allows(name) == (
+ install_llama_prebuilt._vulkan_loader_allows(name)
+ ), (name, pattern, var)
+ finally:
+ os.environ.pop(var, None)
+
+
+def _blocks_under_selector(
+ monkeypatch,
+ value,
+ *,
+ count,
+ var = "HIP_VISIBLE_DEVICES",
+ also = None,
+):
+ """Whether a closed render node blocks a HIP runtime, with one sibling open.
+
+ ``also`` sets a second selector, since which of two the runtime reads is itself a
+ question here."""
+ _nodes(
+ monkeypatch,
+ present = ["/dev/dri/renderD128", "/dev/dri/renderD129", "/dev/kfd"],
+ openable = {"/dev/dri/renderD129", "/dev/kfd"},
+ )
+ monkeypatch.setattr(amd, "amd_kfd_gpu_node_count", lambda: count)
+ for _name in (
+ "HIP_VISIBLE_DEVICES",
+ "ROCR_VISIBLE_DEVICES",
+ "CUDA_VISIBLE_DEVICES",
+ "GPU_DEVICE_ORDINAL",
+ ):
+ monkeypatch.delenv(_name, raising = False)
+ if value is not None:
+ monkeypatch.setenv(var, value)
+ for _name, _value in (also or {}).items():
+ monkeypatch.setenv(_name, _value)
+ return amd.amd_closed_nodes_block_the_runtime()
+
+
+# fmt: off
[email protected]("value, count, var, blocks", _cases(
+ ("a_selector_naming_every_gpu_leaves_the_open_sibling_as_evidence", "0,1", 2, _HIP, False),
+ ("a_selector_naming_one_of_them_still_discards_the_sibling", "0", 2, _HIP, True),
+ ("a_uuid_this_cannot_map_still_discards_it", "GPU-abcdef0123456789", 2, _HIP, True),
+ ("an_unreadable_gpu_count_does_too", "0,1", None, _HIP, True),
+ ("no_selector_at_all_still_keeps_the_sibling", None, 2, _HIP, False),
+ ("a_repeated_rocr_token_ends_the_list_and_narrows", "0,0,1", 2, "ROCR_VISIBLE_DEVICES", True),
+ ("the_same_repeat_under_hip_does_not_narrow", "0,0,1", 2, _HIP, False),
+))
+# fmt: on
+def test_whether_a_selector_narrows_the_host(monkeypatch, linux, value, count, var, blocks):
+ """HIP_VISIBLE_DEVICES=0,1 on a two-GPU host selects the whole host, so the open sibling
+ is still a complete ROCm path and the closed node is a second finding rather than the
+ cause; reading any selector as a narrowing handed that host the group repair in place of
+ the driver diagnosis, and with nothing set the sibling was always evidence. The rule
+ exists because nothing here maps a render node back to the index a selector chose it by,
+ so under a real narrowing the open node may be the excluded GPU's. A UUID names a device
+ by identity and an unreadable GPU count answers nothing: both leave the selector unmapped,
+ and unmapped goes on meaning narrowed.
+
+ ROCr's filter terminates on a token naming a device it has already selected, so
+ ROCR_VISIBLE_DEVICES=0,0,1 surfaces ONE GPU on a two-GPU host; counting distinct ordinals
+ read that as the whole host and kept the sibling as evidence for a runtime that can no
+ longer reach it. The rule is per layer rather than global, because clr's parser stops only
+ on a token that is not its own index written back out, so it accepts the repeat and both
+ GPUs survive -- a global repeat rule would hand that host the group repair instead of the
+ driver diagnosis. Fails before the fix, which asked only whether a selector was set, and
+ accumulated a set with no repeat rule."""
+ assert _blocks_under_selector(monkeypatch, value, count = count, var = var) is blocks
+
+
+# fmt: off
[email protected]("kwargs, contains, absent", _cases(
+ ("a_hybrid_host_is_not_told_to_repair_the_card_its_bundle_will_not_use",
+ dict(torch_index = _CPU_INDEX, nvidia = True), None, []),
+ ("the_same_host_with_an_explicit_rocm_request_is_still_told",
+ dict(torch_index = _CPU_INDEX, backend = "rocm", nvidia = True), [_CLOSED], []),
+ ("an_amd_only_host_on_the_same_route_is_still_told", dict(torch_index = _CPU_INDEX), [_CLOSED],
+ []),
+ ("a_rocm_torch_index_ignores_the_nvidia_card_entirely", dict(nvidia = True), [_CLOSED], []),
+))
+# fmt: on
+def test_whether_the_installer_repairs_a_card_the_run_will_not_use(
+ tmp_path, kwargs, contains, absent
+):
+ """An unset or `auto` backend is resolved by _linux_published_attempts, which takes the
+ CUDA bundle under `if host.has_usable_nvidia:` and reaches ROCm only in the `elif
+ host.has_rocm` below it. So on a hybrid box with a usable NVIDIA GPU neither a CPU torch
+ nor the automatic bundle opens an AMD node, and the AMD-evidence gates cannot tell -- the
+ card is there and its nodes are shut, they are just unused. An explicit rocm request IS a
+ decision and the resolver honours it, so the nodes that bundle opens are the user's
+ problem to repair whatever else is on the bus; with no NVIDIA GPU the automatic bundle may
+ well be the ROCm one, which is the #10466 host, so nothing about `auto` is a reason to go
+ quiet; and a run installing ROCm wheels opens AMD nodes whatever bundle is chosen later,
+ so the bundle question is never reached. Fails before the fix, which read an unresolved
+ `auto` as "may open"."""
+ node, _group = _a_node_a_membership_would_open(tmp_path)
+ _says(_install_sh_hint(str(node), **kwargs), contains, absent)
+
+
+# fmt: off
[email protected]("kwargs, contains, absent", _cases(
+ ("a_uid_with_no_passwd_entry_is_not_handed_a_usermod", dict(id_user = None, env_user = "root"),
+ ["--group-add"], ["usermod -a -G", "root"]),
+ ("an_account_the_system_knows_still_gets_the_command", {}, ["sudo usermod -a -G"],
+ ["--group-add"]),
+ ("and_the_group_sentence_comes_with_it_in_one_piece", {},
+ ["Add yourself to the", "usermod -a -G"], []),
+ ("the_unnamed_gid_repair_drops_its_groupadd_half_too",
+ dict(id_user = None, env_user = "root", repairs = "gid:993"), ["--group-add 993"],
+ ["usermod -a -G", "groupadd -g"]),
+))
+# fmt: on
+def test_the_installer_only_names_an_account_the_system_knows(tmp_path, kwargs, contains, absent):
+ """The shell half of the same item. `id -un` fails outright for a uid the passwd database
+ does not know, and the old fallback then named the inherited $USER -- which in a container
+ commonly still says root, so the command would succeed against an identity nothing is
+ running as and leave the node shut. With a passwd entry the command is the repair, exactly
+ as before. The unnamed-GID branch prints a groupadd AND a usermod, and the second needs
+ the same account, so with no passwd entry the container flag is the whole repair. The
+ last two cases are the control for the ordinary host: a node whose group grants read and
+ write gets the sentence and the command, in one piece. Fails before the fix, which fell
+ back to ${USER}."""
+ node, _group = _a_node_a_membership_would_open(tmp_path)
+ _says(_install_sh_hint(str(node), **kwargs), contains, absent)
+
+
+def _assert_amd_only_loader(reason: str, amd_only: bool) -> None:
+ """The two verdicts a loader configuration reaches for a host with the AMD node shut.
+
+ AMD-only means the other vendor's open node is not a path this binary has, so the node
+ repair stands; anything else leaves the empty probe explained without it.
+ """
+ if amd_only:
+ assert "the Vulkan probe reported no device" not in reason
+ assert "usermod" in reason
+ else:
+ assert reason.startswith("the Vulkan probe reported no device")
+
+
+# Every loader override an arm below may have to clear before stating its own.
+_VK_OVERRIDE_VARS = (
+ "VK_DRIVER_FILES",
+ "VK_ICD_FILENAMES",
+ "VK_ADD_DRIVER_FILES",
+ "VK_LOADER_DRIVERS_SELECT",
+ "VK_LOADER_DRIVERS_DISABLE",
+)
+
+
+def _assert_scope(
+ out: str,
+ *,
+ silent = False,
+ present = (),
+ absent = (),
+):
+ """What the KFD-scoped closed-node message did and did not say."""
+ if silent:
+ assert out.strip() == ""
+ for _text in present:
+ assert _text in out, _text
+ for _text in absent:
+ assert _text not in out, _text
+
+
+_UNDER_CUDA = {"also": {"CUDA_VISIBLE_DEVICES": "0"}}
+
+
+# fmt: off
+ pytest.param(("0,1", _UNDER_CUDA, False), id = "a_cuda_selector_under_a_hip_one"),
+ pytest.param((None, _UNDER_CUDA, True), id = "a_cuda_selector_on_its_own"),
+ pytest.param(("0,1,-1", {}, False), id = "a_list_terminated_by_an_unmappable_token"),
+ pytest.param(("0,1,later", {}, False), id = "a_list_terminated_by_a_word"),
+ pytest.param(("0,-1", {}, True), id = "a_prefix_that_stops_short"),
+ pytest.param(("0,0,1", {"var": "ROCR_VISIBLE_DEVICES"}, True),
+ id = "a_repeat_in_the_rocr_list"),
+])
+# fmt: on
+def test_which_selector_lists_narrow_a_two_gpu_host(monkeypatch, linux, case):
+ """Whether a closed render node blocks the runtime, for each shape of selector list.
+
+ The HIP layer reads HIP_VISIBLE_DEVICES when non-empty and CUDA_VISIBLE_DEVICES only
+ otherwise, so a CUDA value under a HIP one narrows nothing while the same value alone IS
+ the selector, the precedence _hip_layer_var already applies; reading all four side by side
+ let the shadowed one discard an open sibling the runtime can still reach. clr BREAKS out
+ of its parse on a token it cannot map, having already pushed what it accepted, so 0,1,-1
+ leaves both GPUs visible and a word ends the list identically (atoi returns 0, which is
+ not the token written back) -- read as narrowed, those hosts got the group repair in place
+ of the driver diagnosis they need. The prefix is judged rather than trusted because 0,-1
+ accepts ONE of two and really is narrowed. ROCr's RvdFilter ends the same way on an
+ already-selected token.
+
+ Fails before the fix, which asked every name independently and returned False on the
+ unmappable token."""
+ value, extra, blocks = case
+ assert _blocks_under_selector(monkeypatch, value, count = 2, **extra) is blocks
+
+
+def _two_vendor_icd_dir(tmp_path) -> "list[str]":
+ """One search directory holding a loadable AMD manifest and a loadable foreign one."""
+ _amd = _icd_manifest(tmp_path, "radeon_icd.x86_64.json", library = "libamd.so")
+ _other = _icd_manifest(tmp_path, "nvidia_icd.json", library = "libnv.so")
+ assert _amd and _other
+ return [str(tmp_path)]
+
+
+# fmt: off
+ pytest.param(({"VK_LOADER_DRIVERS_SELECT": "radeon*"}, True), id = "filtered_to_amd"),
+ pytest.param((None, False), id = "the_same_two_drivers_unfiltered"),
+])
+# fmt: on
+def test_a_loader_search_is_read_through_its_filters(monkeypatch, linux, tmp_path, case):
+ """VK_LOADER_DRIVERS_SELECT applies to whatever the loader would load, forced list or
+ search, so a host with no list at all can still be pinned to AMD alone; reading the two
+ force-list variables only left it crediting the other vendor's open node to a binary whose
+ loader never opens that vendor's driver. Unfiltered the loader loads both manifests, so
+ that node IS a path and the closed AMD one cannot be why the probe was empty -- otherwise
+ the rule becomes "a search always means AMD only", which suppresses the finding on every
+ host.
+
+ Fails before the fix, which asked what a forced list was named rather than what the
+ loader would load."""
+ filters, amd_only = case
+ reason = _vulkan_reason_under_icd_list(
+ monkeypatch, None, search_dirs = _two_vendor_icd_dir(tmp_path), filters = filters
+ )
+ _assert_amd_only_loader(reason, amd_only)
+
+
+def test_a_search_that_enumerates_nothing_answers_nothing(monkeypatch, linux, tmp_path):
+ """Positive evidence only. An empty search is not "AMD alone", it is a loader this cannot
+ read -- and a loader with no driver explains the empty probe by itself, so the closed AMD
+ node is not the cause either and must not be suppressed."""
+ reason = _vulkan_reason_under_icd_list(
+ monkeypatch, None, search_dirs = [str(tmp_path / "nothing-here")]
+ )
+ assert reason.startswith("the Vulkan probe reported no device")
+
+
+def test_the_search_dirs_follow_the_xdg_variables(monkeypatch):
+ """The loader falls back to its defaults only when a variable is unset, so reading the
+ defaults regardless both misses a custom layout's only manifest and counts stale ones the
+ loader would never read. The test below holds this list and the installer's together."""
+ monkeypatch.setenv("XDG_DATA_DIRS", "/opt/one:/opt/two")
+ monkeypatch.setenv("XDG_CONFIG_DIRS", "/opt/conf")
+ dirs = amd._vulkan_icd_search_dirs()
+ assert "/opt/one/vulkan/icd.d" in dirs
+ assert "/opt/two/vulkan/icd.d" in dirs
+ assert "/opt/conf/vulkan/icd.d" in dirs
+ assert "/usr/share/vulkan/icd.d" not in dirs
+ assert "/etc/xdg/vulkan/icd.d" not in dirs
+ assert dirs.count("/etc/vulkan/icd.d") == 1
+
+
+def test_the_search_dirs_match_the_installers(monkeypatch):
+ """One loader, one question, so the two lists are one list: a copy that drifts sends the
+ two halves of this diagnosis to different drivers."""
+ import install_llama_prebuilt
+
+ monkeypatch.setenv("XDG_DATA_DIRS", "/opt/one:/opt/two")
+ monkeypatch.setenv("XDG_CONFIG_DIRS", "/opt/conf")
+ assert amd._vulkan_icd_search_dirs() == [
+ str(directory) for directory in install_llama_prebuilt._vulkan_icd_search_dirs()
+ ]
+
+
+# fmt: off
+ pytest.param(("vul kan", True, True, ()), id = "a_rejected_value_on_a_hybrid_box"),
+ pytest.param(("rocm", True, False, ("cannot open its device nodes", "/dev/kfd")),
+ id = "an_explicit_rocm_request_on_the_same_box"),
+ pytest.param(("vul kan", False, False, ("/dev/kfd",)),
+ id = "the_rejected_value_on_an_amd_only_box"),
+])
+# fmt: on
+def test_where_the_automatic_route_reports_a_closed_kfd(case):
+ """setup.sh warns "Ignoring UNSLOTH_LLAMA_CPP_BACKEND=..." for anything outside its list
+ and the installer normalises it to auto, so a typo installs the automatically chosen
+ bundle -- on a hybrid box CUDA, which opens no AMD node. Listing the two spellings of "no
+ decision" instead of the values that ARE decisions let a rejected value pose as one. An
+ explicit rocm request is honoured by the resolver, so its bundle opens /dev/kfd on a
+ hybrid box exactly as on an AMD-only one, and reading every hybrid host as CUDA would
+ silence #10466 for the users who asked for ROCm. The automatic route is silent only where
+ it resolves away from AMD: with no NVIDIA card it resolves to ROCm and reports.
+
+ Fails before the fix, which matched "" and auto alone."""
+ backend, nvidia, silent, present = case
+ out = _install_sh_kfd_scope("/dev/kfd", skip_torch = True, backend = backend, nvidia = nvidia)
+ _assert_scope(out, silent = silent, present = present)
+
+
+def _nvidia_probe_calls(backend = None):
+ """How many times the two scope predicates run the NVIDIA probe for one install."""
+ lines = _install_sh_lines()
+ script = "\n".join(
+ [
+ "SKIP_TORCH=true",
+ "TORCH_INDEX_URL=''",
+ f"export UNSLOTH_LLAMA_CPP_BACKEND={backend or ''!r}",
+ "_probe_calls=0",
+ "_has_usable_nvidia_gpu() { _probe_calls=$((_probe_calls + 1)); return 1; }",
+ # Every one of these is lifted rather than stubbed, for the reasons
+ # _install_sh_hint gives: a harness that omits one measures a missing function
+ # rather than a rule, and _shell_quote left out makes every interpolated name
+ # come back EMPTY.
+ *(
+ _shell_fn(lines, _name)
+ for _name in (
+ "_torch_index_url_leaf",
+ "_is_pip_rocm_family_leaf",
+ "_requested_llama_backend",
+ "_torch_opens_amd_nodes",
+ "_auto_bundle_opens_amd_nodes",
+ "_run_may_open_kfd",
+ "_shell_quote",
+ "_run_may_open_a_gpu_node",
+ )
+ ),
+ "for _i in 1 2 3 4; do _run_may_open_kfd; _run_may_open_a_gpu_node; done",
+ 'echo "$_probe_calls"',
+ ]
+ )
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True, check = True)
+ return int(out.stdout.strip())
+
+
+# fmt: off
+ pytest.param((None, 1), id = "the_automatic_route_probes_once"),
+ pytest.param(("rocm", 0), id = "an_explicit_backend_never_probes_at_all"),
+])
+# fmt: on
+def test_how_often_the_nvidia_probe_runs(case):
+ """_has_usable_nvidia_gpu shells out to a bounded `nvidia-smi -L` on every call and is not
+ memoized, while the two scope predicates are consulted at every diagnosis; nothing it
+ reads changes within a run, so an NVIDIA-less host was paying a subprocess per gate. A
+ decision the resolver honours settles the question without asking about the other vendor's
+ hardware at all, so the memo is not merely cheaper there, it is unreached.
+
+ Fails before the fix, which probed on every call."""
+ backend, calls = case
+ assert _nvidia_probe_calls(backend) == calls
+
+
+# fmt: off
+ pytest.param(("rocm", True), id = "an_explicit_rocm_bundle"),
+ pytest.param(("vulkan", True), id = "an_explicit_vulkan_bundle"),
+ pytest.param(("cpu", False), id = "a_cpu_bundle"),
+ pytest.param((None, False), id = "no_request_at_all"),
+])
+# fmt: on
+def test_whether_an_explicit_bundle_routes_the_node_diagnoses(case):
+ """The route was derived from the torch index alone, so a CUDA-pinned index asked for the
+ ROCm bundle read as a CUDA route and silenced all three diagnoses -- for a run whose bundle
+ opens the very nodes they are about. The SKIP_TORCH override below the case could not
+ catch it, since it only runs when no wheel is installed at all. The new arm only ever
+ turns the route ON, so a cpu request and no request at all stay with the two scope
+ predicates, which is where a bundle that opens no AMD node belongs.
+
+ Fails before the fix, which had no arm for the backend request here."""
+ backend, routed = case
+ assert _diag_route("https://download.pytorch.org/whl/cu128", backend = backend) is routed
+
+
+# fmt: off
+ pytest.param((True, False), id = "a_hybrid_host_rocm_can_see"),
+ pytest.param((False, True), id = "a_hybrid_host_rocm_cannot_see"),
+])
+# fmt: on
+def test_the_kernel_stack_hint_on_a_hybrid_rocm_host(tmp_path, case):
+ """_has_amd_rocm_gpu opens with `if _has_usable_nvidia_gpu; then return 1`, right where it
+ is choosing a torch index and wrong here: this branch has already established the run
+ opens AMD nodes, so the veto made rocminfo's answer unreachable and a healthy hybrid ROCm
+ host was told to install the ROCm kernel stack it already has. An AMD card on the bus that
+ ROCm genuinely cannot see is what the sentence is for, so dropping the veto must not have
+ dropped the finding.
+
+ Fails before the fix, which read the wrapped probe."""
+ rocm_visible, says_rocm_cannot_see_it = case
+ out = _install_sh_missing_kfd(
+ tmp_path, topology = False, nvidia = True, backend = "rocm", rocm_visible = rocm_visible
+ )
+ assert ("ROCm cannot see it" in out) is says_rocm_cannot_see_it
+
+
+def _search_plus_added_driver(tmp_path, name, library):
+ """An AMD-only search directory, and one more manifest registered from outside it."""
+ search = tmp_path / "icd.d"
+ search.mkdir()
+ amd_manifest = _icd_manifest(search, "radeon_icd.x86_64.json", library = "libamd.so")
+ elsewhere = tmp_path / "vendor"
+ elsewhere.mkdir()
+ return str(search), amd_manifest, _icd_manifest(elsewhere, name, library = library)
+
+
+# fmt: off
+ pytest.param(("nvidia_icd.json", "libnv.so", False, False), id = "a_foreign_driver_added"),
+ pytest.param(("nvidia_icd.json", "libnv.so", True, True),
+ id = "the_same_one_under_a_forced_list"),
+ pytest.param(("amdvlk64.json", "libamdvlk.so", False, True), id = "an_amd_driver_added"),
+])
+# fmt: on
+def test_how_an_added_driver_list_is_read(monkeypatch, linux, tmp_path, case):
+ """VK_ADD_DRIVER_FILES is read FIRST and then the search, and it may name a manifest no
+ search directory holds; leaving it out made a host whose only foreign driver arrived that
+ way look AMD-only, and that vendor's open node then stopped excusing the closed AMD one.
+ The loader ignores the additive list entirely when VK_DRIVER_FILES or VK_ICD_FILENAMES is
+ set, so a foreign driver added beside a forced AMD list is not a path this binary has; and
+ reading the list must not make every host that has one look mixed-vendor, so an AMD
+ manifest added to an AMD-only search is still AMD alone.
+
+ Fails before the fix, which read the search alone."""
+ name, library, forced, amd_only = case
+ search, amd_manifest, added = _search_plus_added_driver(tmp_path, name, library)
+ reason = _vulkan_reason_under_icd_list(
+ monkeypatch,
+ amd_manifest if forced else None,
+ search_dirs = [search],
+ filters = {"VK_ADD_DRIVER_FILES": added},
+ )
+ _assert_amd_only_loader(reason, amd_only)
+
+
+# fmt: off
+ pytest.param(({"vendor_readable": False}, None, _AMD_NODES, "/dev/dri/renderD128"),
+ id = "a_hidden_vendor_over_an_amd_topology"),
+ pytest.param(({"vendor_readable": False, "amd_owned": False}, None, [], None),
+ id = "a_hidden_vendor_over_a_foreign_topology"),
+ pytest.param(({}, "0x10de", ["/dev/kfd"], None), id = "a_readable_foreign_vendor"),
+ pytest.param(({"topology": None}, None, _AMD_NODES, None),
+ id = "a_hidden_topology_drm_confirms"),
+ pytest.param(({"amd_owned": False, "topology": False}, None, [], None),
+ id = "a_readable_topology_naming_no_amd_gpu"),
+ pytest.param(({"vendor_readable": False, "topology": None}, None, [], None),
+ id = "a_hidden_topology_drm_cannot_confirm"),
+])
+# fmt: on
+def test_which_shut_nodes_are_credited_to_amd(monkeypatch, linux, case):
+ """Which of the two shut nodes reaches the closed list, for each shape of evidence.
+
+ A container can map the render node and hide the sysfs entry naming its vendor. Requiring
+ a confirmed AMD vendor dropped the node, so nothing was CLOSED -- while
+ _amd_render_node_exists reads the same unknown as PRESENT and withdraws the missing-node
+ sentence too, leaving #10466's own shape with no diagnosis at all. KFD is the independent
+ evidence that keeps this off every NVIDIA host, since its topology reports 0x10DE there,
+ and a vendor that CAN be read and is not AMD is positive evidence the node is somebody
+ else's even where the topology does report an AMD GPU.
+
+ The same distinction one node over: a container can map /dev/kfd and hide /sys/class/kfd,
+ and the guard read that as "not an AMD GPU". DRM says otherwise and independently, so
+ /dev/kfd was dropped from a list the render node stayed in -- and where the two carry
+ different owning groups the hint then named a membership that leaves KFD shut and ROCm
+ with nothing to open. A topology that READS and names no AMD GPU is positive evidence
+ rather than absence, so the fallback must not fire there; and it is vendor-CONFIRMED,
+ never assumed, so an unreadable render vendor cannot stand in for a topology this could
+ not read either.
+
+ Fails before the fix, which asked for a vendor the container had hidden."""
+ node_kwargs, vendor, closed, hint = case
+ _nodes(monkeypatch, present = _AMD_NODES, openable = set(), **node_kwargs)
+ if vendor is not None:
+ monkeypatch.setattr(amd, "_render_node_vendor", lambda path: vendor)
+ assert amd.amd_nodes_closed_to_this_user() == closed
+ if hint is not None:
+ assert hint in amd.amd_node_permission_hint()
+
+
+_CUDA_WHEEL = "https://download.pytorch.org/whl/cu128"
+_RADEON_WHEEL = "https://repo.radeon.com/rocm/manylinux/rocm-rel-7.0/gfx1151"
+
+
+# fmt: off
+ pytest.param(("/dev/kfd", "vulkan", _CUDA_WHEEL, True, True, (), ()),
+ id = "a_cuda_wheel_beside_a_vulkan_bundle"),
+ pytest.param(("/dev/kfd\n/dev/dri/renderD128", "vulkan", _CUDA_WHEEL, True, False,
+ ("/dev/dri/renderD128",), ("/dev/kfd",)),
+ id = "the_same_pair_with_a_closed_render_node"),
+ pytest.param(("/dev/kfd", "rocm", _CUDA_WHEEL, True, False, ("/dev/kfd",), ()),
+ id = "a_cuda_wheel_asking_for_the_rocm_bundle"),
+ pytest.param(("/dev/kfd", "vulkan", _RADEON_WHEEL, False, False, ("/dev/kfd",), ()),
+ id = "a_radeon_repo_wheel_under_a_vulkan_bundle"),
+])
+# fmt: on
+def test_which_nodes_reach_the_kfd_scope(case):
+ """Which closed nodes survive the KFD filter, for a wheel index and a bundle request.
+
+ Only a ROCm wheel opens /dev/kfd, and reading "any index that is not cpu" as one that does
+ was harmless only while _amd_node_diag_route dropped every non-ROCm index; the
+ explicit-backend arm keeps the route for a vulkan request, so a CUDA wheel beside a Vulkan
+ bundle reached the KFD scope and told a healthy hybrid host to repair a node neither opens.
+ That bundle DOES open a render node, so scoping /dev/kfd must not take the diagnosis the
+ explicit-backend arm exists to reach (the round-twenty-four fix), and the bundle is the
+ other half of the question, so an explicit rocm request keeps /dev/kfd even though the CUDA
+ wheel never touches it. On the route this installer reroutes #10466's own host to,
+ repo.radeon.com's leaf is rocm-rel-X.Y, not a pip family: a classifier knowing only the pip
+ spellings silences the diagnosis for the host it was written for.
+
+ Fails before the fix, which excluded the cpu leaf alone."""
+ nodes, backend, wheel, nvidia, silent, present, absent = case
+ out = _install_sh_kfd_scope(
+ nodes, skip_torch = False, backend = backend, torch_index = wheel, nvidia = nvidia
+ )
+ _assert_scope(out, silent = silent, present = present, absent = absent)
+
+
+# fmt: off
+ pytest.param(([("radeon_icd.i686.json", None)], False), id = "a_32_bit_amd_registration"),
+ pytest.param(([("radeon_icd.i686.json", None), ("radeon_icd.x86_64.json", None)], True),
+ id = "the_multilib_pair"),
+ pytest.param(([("nvidia_icd.i686.json", "libGLX_nvidia32.so"),
+ ("radeon_icd.x86_64.json", None)], True),
+ id = "a_32_bit_foreign_manifest_beside_an_amd_one"),
+ pytest.param(([("nvidia_icd.json", "libGLX_nvidia.so"),
+ ("radeon_icd.x86_64.json", None)], False),
+ id = "a_64_bit_foreign_manifest_beside_an_amd_one"),
+])
+# fmt: on
+def test_which_registrations_a_64_bit_binary_can_load(monkeypatch, linux, tmp_path, case):
+ """A distribution registers the 32-bit ICD beside the 64-bit one, and a 64-bit
+ llama-server cannot load it: a host left with only that has no driver at all, so the other
+ vendor's open node is not shut out by an AMD-only loader -- there is no loader. The pair is
+ the layout every multilib host has, one AMD driver registered twice, so the loader still
+ loads AMD and only AMD; otherwise the rule becomes "any i686 name means no AMD driver",
+ which is the normal case. Asked of every vendor rather than of AMD alone, because treating
+ a 32-bit manifest as simply "not AMD" would make nvidia_icd.i686.json evidence that this
+ loader reaches another vendor's card, when a 64-bit binary can load neither it nor the
+ card behind it. The same foreign driver in its loadable build IS such a path, so the
+ exclusion cannot be "ignore other vendors".
+
+ Fails before the fix, which read the name as AMD and suppressed."""
+ manifests, amd_only = case
+ paths = [
+ _icd_manifest(tmp_path, _name, **({"library": _library} if _library else {}))
+ for _name, _library in manifests
+ ]
+ reason = _vulkan_reason_under_icd_list(monkeypatch, os.pathsep.join(paths))
+ _assert_amd_only_loader(reason, amd_only)
+
+
+def test_the_32_bit_rule_matches_the_installers(tmp_path):
+ """The installer decides whether an AMD driver is installed from the same filenames, and
+ a host where the two disagree gets one answer from the Vulkan route and another from this
+ diagnosis. Only the needles are shared: the installer asks it of AMD names alone, this of
+ every vendor, which is a difference in question not in rule."""
+ import install_llama_prebuilt
+
+ assert set(amd._VULKAN_ICD_32_BIT_NEEDLES) == set(
+ install_llama_prebuilt._AMD_VULKAN_ICD_32_BIT_NEEDLES
+ )
+ for name in ("radeon_icd.i686.json", "radeon_icd.i386.json", "radeon_icd32.json"):
+ assert amd._is_a_32_bit_icd_name(name), name
+ for name in ("radeon_icd.x86_64.json", "radeon_icd.aarch64.json"):
+ assert not amd._is_a_32_bit_icd_name(name), name
+
+
+def _vulkan_node_hint_under_icd_list(
+ monkeypatch,
+ value,
+ *,
+ search_dirs = None,
+ env = None,
+ sibling_open = False,
+):
+ """The empty-probe reason for a Vulkan build with the AMD node shut, so the answer is the
+ node repair and the question is what is appended to it.
+
+ ``sibling_open`` is another vendor's node being open, which demotes the node repair to a
+ second finding; the default is no such sibling.
+ """
+ _nodes(monkeypatch, present = _AMD_NODES, openable = set())
+ monkeypatch.setattr(amd, "a_non_amd_render_node_is_open", lambda: sibling_open)
+ for _var in _VK_OVERRIDE_VARS:
+ monkeypatch.delenv(_var, raising = False)
+ if search_dirs is not None:
+ monkeypatch.setattr(amd, "_vulkan_icd_search_dirs", lambda: list(search_dirs))
+ if value is not None:
+ monkeypatch.setenv("VK_DRIVER_FILES", value)
+ # After the clearing above, since that is what makes an arm about ONE override able to
+ # set it. `value` reaches the loader through VK_DRIVER_FILES, so an arm that is about a
+ # different override passes None and names its search dirs instead.
+ for _name, _value in (env or {}).items():
+ monkeypatch.setenv(_name, _value)
+ _ggml(monkeypatch, {"vulkan"})
+ return _empty_probe()
+
+
+def test_the_no_driver_diagnosis_stays_primary_when_a_sibling_node_is_open(
+ monkeypatch, linux, tmp_path
+):
+ """Codex 4040623258. With another vendor's node open, the closed AMD node is a SECOND
+ finding: the runtime had a complete path and enumerated nothing anyway. The loader having
+ no loadable driver is not second, it is sufficient on its own, and folding it into
+ node_hint demoted it along with the permission text to "not why the probe is empty" -- the
+ one sentence that always holds, filed under the one that does not."""
+ _icd_manifest(tmp_path, "radeon_icd.json", present = False)
+ reason = (
+ _vulkan_node_hint_under_icd_list(
+ monkeypatch, None, search_dirs = [str(tmp_path)], sibling_open = True
+ )
+ or ""
+ )
+ assert "no driver it can load" in reason
+ assert "reinstall the Vulkan driver" in reason
+ # The demotion applies to the permission finding only, so the loader sentence must come
+ # BEFORE it rather than inside it.
+ _demoted = reason.index("Separately, and not why the probe is empty")
+ assert reason.index("no driver it can load") < _demoted
+ # And "also" is dropped, since no node repair precedes it here.
+ assert "loader also has no driver" not in reason
+
+
+# fmt: off
+ pytest.param((lambda tmp: {"value": _icd_manifest(tmp, "radeon_icd.json", present = False)},
+ True),
+ id = "a_registration_with_no_library_behind_it"),
+ pytest.param((lambda tmp: {"value": None, "search_dirs": [str(tmp / "empty")]}, False),
+ id = "a_loader_that_could_not_be_enumerated"),
+ pytest.param((lambda tmp: {"value": _icd_manifest(tmp, "radeon_icd.x86_64.json")}, False),
+ id = "a_loadable_driver"),
+])
+# fmt: on
+def test_when_the_no_driver_sentence_joins_the_node_repair(monkeypatch, linux, tmp_path, case):
+ """Two things are wrong at once and the node repair clears only one of them: with every
+ registered manifest unloadable the probe stays empty however the node is owned, so a user
+ who runs the usermod and nothing else is left exactly where they were. Fail-closed in the
+ other direction, which is the one that matters here: an enumeration that found NO
+ manifests means this could not read the loader's configuration, not that the loader has
+ nothing, and emitting the sentence there sends a working host after a driver reinstall it
+ does not need on top of a repair it does. Manifests found AND loadable is the ordinary
+ host, where the closed node is the whole story and the sentence must stay off."""
+ configuration, says_no_driver = case
+ _kwargs = configuration(tmp_path)
+ reason = _vulkan_node_hint_under_icd_list(monkeypatch, _kwargs.pop("value"), **_kwargs)
+ assert "usermod" in reason
+ assert ("no driver it can load" in reason) is says_no_driver
+
+
+# fmt: off
+ pytest.param((39, {39: "render"}, True, {"already": ["render"], "joinable": []}),
+ id = "a_named_group_this_account_holds"),
+ pytest.param((39, {39: "render"}, False, {"joinable": ["render"], "already": []}),
+ id = "a_named_group_this_account_is_outside"),
+ pytest.param((993, {}, True, {"already": ["993"], "unnamed": []}),
+ id = "an_unnamed_gid_this_account_holds"),
+ pytest.param((993, {}, False, {"unnamed": [993], "already": []}),
+ id = "an_unnamed_gid_this_account_lacks"),
+])
+# fmt: on
+def test_which_bucket_an_owning_group_lands_in(monkeypatch, linux, case):
+ """os.access already said the node is shut, so a group this account is ALREADY in is not
+ what is denying it: usermod exits 0 and leaves the node exactly as closed, and the denial
+ is outside the file mode -- a container device cgroup, or an LSM. A group it does not hold
+ IS the repair, and is what #10466 asked for, so the rule cannot be "never prescribe a
+ group". The same one branch over: a numeric owner with no group-database entry is the
+ container shape, where the repair is groupadd plus --group-add, itself the same empty
+ promise once the account carries the gid -- which is why the already-held question is
+ asked BEFORE the naming branch.
+
+ Fails before the fix, which read the group bits and prescribed the membership."""
+ gid, names, held, expected = case
+ _stat_nodes(monkeypatch, {"/dev/kfd": (gid, 0o660, 0)}, names)
+ monkeypatch.setattr(amd.os, "getgroups", lambda: [gid] if held else [])
+ monkeypatch.setattr(amd.os, "getgid", lambda: gid if held else 1)
+ _got = dict(zip(_BUCKETS, amd._groups_that_own(["/dev/kfd"])))
+ for _bucket, _value in expected.items():
+ assert _got[_bucket] == _value, _bucket
+
+
+def test_the_hint_for_a_group_already_held_names_the_cgroup_instead(monkeypatch, linux):
+ """The sentence a user actually reads, since the buckets above only decide it: no usermod,
+ and a statement of what is left to look at."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ _owning(monkeypatch, already = ["render"])
+ hint = amd.amd_node_permission_hint()
+ # The command, not the word: the sentence itself says usermod would change nothing.
+ assert "usermod -a -G" not in hint
+ assert "already in the render group" in hint
+ assert "cgroup" in hint
+
+
+# fmt: off
+ pytest.param((lambda gid: str(gid), True), id = "the_owning_gid_itself"),
+ pytest.param((lambda gid: str(gid + 1), False), id = "a_neighbouring_gid"),
+ pytest.param((lambda gid: f"{gid}7 {gid}9", False), id = "two_gids_it_is_a_prefix_of"),
+])
+# fmt: on
+def test_whether_the_installer_prescribes_the_owning_group(tmp_path, case):
+ """The shell half of the rule above, read from `id -G`; without it the two halves disagree
+ on the same host and the installer prints the usermod the Python side just declined to.
+ The control is an account outside the group, where the command comes back, so the shell
+ rule cannot be "never prescribe". The trap in reading `id -G` as text is that the list is
+ space separated, so an unpadded match makes gid 100 look held by an account that is only
+ in 1001: the last case is owned by a group the account does NOT have, whose gid is a
+ prefix of two it does."""
+ gids_for, already = case
+ node, _group = _a_node_a_membership_would_open(tmp_path)
+ out = _install_sh_hint(str(node), self_gids = gids_for(os.stat(node).st_gid))
+ assert ("already in the" in out) is already
+ assert ("sudo usermod -a -G" in out) is (not already)
+ if already:
+ assert "cgroup" in out
+
+
+def _bare_soname_manifest(
+ tmp_path,
+ name,
+ soname = "libvulkan_radeon.so",
+):
+ """An ICD manifest naming its library by soname alone, and the path to it.
+
+ The form NVIDIA registers under, and the one _icd_manifest cannot produce: that helper
+ writes an absolute path so it can put the library on disk, which is exactly the case this
+ is not.
+ """
+ path = tmp_path / name
+ path.write_text(
+ json.dumps(
+ {
+ "file_format_version": "1.0.0",
+ "ICD": {"library_path": soname, "api_version": "1.3.0"},
+ }
+ ),
+ encoding = "utf-8",
+ )
+ return str(path)
+
+
+_RADEON_SONAME = "libvulkan_radeon.so"
+_NVIDIA_SONAME = "libGLX_nvidia.so.0"
+
+
+# fmt: off
+ pytest.param(("radeon_icd.json", _RADEON_SONAME, False, frozenset({"libc.so.6"}), False),
+ id = "a_soname_nothing_can_resolve"),
+ pytest.param(("radeon_icd.json", _RADEON_SONAME, True, frozenset(), True),
+ id = "a_soname_on_the_search_path"),
+ pytest.param(("nvidia_icd.json", _NVIDIA_SONAME, False, frozenset({_NVIDIA_SONAME}), True),
+ id = "a_soname_only_the_loader_cache_knows"),
+ pytest.param(("nvidia_icd.json", _NVIDIA_SONAME, False, None, True),
+ id = "a_soname_under_a_cache_that_cannot_be_read"),
+])
+# fmt: on
+def test_when_a_bare_soname_registration_is_a_driver(monkeypatch, linux, tmp_path, case):
+ """A manifest may name its library by soname and leave the loader to find it, so the
+ package can be removed and leave the registration behind; trusting the name counted a
+ driver that is not there and withheld the reinstall half of the repair. The control is the
+ same manifest with the library where ld.so would find it, since a bare name IS the
+ ordinary case for every vendor that registers one. The cache is consulted because a
+ versioned soname such as libGLX_nvidia.so.0 lives wherever ld.so.conf put it, which need
+ not be a directory this enumerates. And it fails closed: musl ships no ldconfig -p and a
+ minimal container may ship none at all, so "not found" there is ignorance rather than
+ absence, and calling a live driver stale would promote the AMD node as the sole cause on a
+ host whose other vendor really does have a path.
+
+ Fails before the fix, which returned True for every bare name."""
+ name, soname, on_disk, cache, usable = case
+ lib = tmp_path / "lib"
+ lib.mkdir()
+ if on_disk:
+ (lib / soname).write_bytes(b"")
+ manifest = _bare_soname_manifest(tmp_path, name, soname)
+ monkeypatch.setattr(amd, "_dynamic_loader_search_dirs", lambda: [str(lib)])
+ monkeypatch.setattr(amd, "_ld_cache_sonames", lambda: cache)
+ assert amd._icd_manifest_is_usable(manifest) is usable
+
+
+def test_the_loader_cache_reader_says_none_rather_than_empty_when_ldconfig_is_gone(
+ monkeypatch, linux
+):
+ """The distinction the arm above rests on, at its source: no ldconfig has to answer None,
+ because an empty set would read as "no library is installed" and call every bare
+ registration on the host stale."""
+ monkeypatch.setattr(amd, "_ld_cache_read", False)
+ monkeypatch.setattr(amd, "_ld_cache_sonames_cached", None)
+ monkeypatch.setattr(amd.shutil, "which", lambda _name: None)
+ monkeypatch.setattr(amd.os.path, "exists", lambda _p: False)
+ assert amd._ld_cache_sonames() is None
+
+
+def test_a_stale_bare_registration_reaches_the_driver_sentence(monkeypatch, linux, tmp_path):
+ """What the classification is for: with the only registration unresolvable the loader has
+ no driver at all, so the node repair alone would leave the probe empty."""
+ manifest = _bare_soname_manifest(tmp_path, "radeon_icd.json")
+ monkeypatch.setattr(amd, "_dynamic_loader_search_dirs", lambda: [str(tmp_path / "lib")])
+ monkeypatch.setattr(amd, "_ld_cache_sonames", lambda: frozenset({"libc.so.6"}))
+ reason = _vulkan_node_hint_under_icd_list(monkeypatch, manifest)
+ assert "usermod" in reason
+ assert "no driver it can load" in reason
+
+
+def test_the_mask_fixture_isolates_every_selector_the_rule_reads():
+ """The fixture decides what "no mask" means for this whole suite, so a selector the
+ production rule reads and the fixture does not clear is answered by the runner's own
+ environment: a ROCm or OpenCL host exports GPU_DEVICE_ORDINAL, and the no-mask controls
+ would then quietly take the narrowing branch they exist to rule out. Read out of the
+ rule's own source rather than restated, so adding a fifth selector fails here instead of
+ drifting."""
+ _source = inspect.getsource(amd._a_per_gpu_mask_narrows_the_runtime)
+ _read = set(re.findall(r'"([A-Z_]+(?:VISIBLE_DEVICES|DEVICE_ORDINAL))"', _source))
+ assert _read, "the rule named no selector, so this test proves nothing"
+ assert _read <= set(_GPU_MASK_VARS), _read - set(_GPU_MASK_VARS)
+
+
+def test_the_installer_chains_the_groupadd_pair(tmp_path):
+ """groupadd and usermod are a pair, and the name is generated from the GID, which says
+ nothing about whether that NAME is free. Printed as two separate lines, a host that
+ already has an amdgpu993 group at another GID fails the groupadd and then SUCCEEDS the
+ usermod against the wrong group, leaving the node shut having reported success. Fails
+ before the fix, which printed them unchained."""
+ out = _install_sh_hint("/dev/dri/renderD128", repairs = "gid:993")
+ _lines = out.splitlines()
+ _at = next(i for i, l in enumerate(_lines) if "groupadd" in l)
+ # && plus a continuation, so the pair pastes as one command across the two lines.
+ assert _lines[_at].rstrip().endswith("&& \\"), _lines[_at]
+ assert "993 amdgpu993" in _lines[_at]
+ assert "usermod -a -G amdgpu993" in _lines[_at + 1]
+
+
+def test_the_python_half_chains_the_groupadd_pair_too(monkeypatch, linux):
+ """Its twin, which already chained: asserted so that the two halves cannot drift apart the
+ way they just did."""
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ _owning(monkeypatch, unnamed = [993])
+ hint = amd.amd_node_permission_hint()
+ assert "groupadd -g 993 amdgpu993 && sudo usermod -a -G amdgpu993" in hint
+
+
+def test_the_named_group_repair_is_still_one_command(tmp_path):
+ """The control: a node whose owning group HAS a name needs no groupadd, so the repair is a
+ single usermod and must not have grown a chain."""
+ out = _install_sh_hint("/dev/dri/renderD128", repairs = "join:render")
+ assert "groupadd" not in out
+ _line = next(l for l in out.splitlines() if "usermod" in l)
+ assert "&&" not in _line
+
+
+def _a_closed_node_file(tmp_path, name):
+ """A file standing in for a device node this account cannot open."""
+ node = tmp_path / name
+ node.write_bytes(b"")
+ node.chmod(0o000)
+ return node
+
+
+def _install_sh_closed_nodes(nodes, *, vendors, topology: bool) -> "list[str]":
+ """The installer's closed-node enumeration, run over a named set of files.
+
+ The real one reads /dev and /sys, which no test can reach, so the two seams it now has
+ are stubbed and the node files themselves are real: the mode tests are the rule under
+ test and must run against actual permissions.
+ """
+ lines = _install_sh_lines()
+ _vendor_cases = " ".join(
+ f"{shlex.quote(str(_path))}) printf %s {shlex.quote(_vendor)} ;;"
+ for _path, _vendor in vendors.items()
+ )
+ script = "\n".join(
+ [
+ "_amd_candidate_nodes() { printf '%s\\n' "
+ + " ".join(shlex.quote(str(_n)) for _n in nodes)
+ + "; }",
+ f"_kfd_topology_has_an_amd_gpu() {{ return {0 if topology else 1}; }}",
+ # A vendor sysfs will not name exits non-zero, which is the case under test.
+ '_amd_render_node_vendor() { case "$1" in ' + _vendor_cases + " *) return 1 ;; esac; }",
+ _shell_fn(lines, "_amd_nodes_closed_to_this_user"),
+ "_amd_nodes_closed_to_this_user",
+ ]
+ )
+ out = subprocess.run(["bash", "-c", script], capture_output = True, text = True)
+ assert out.returncode == 0, out.stderr
+ return [line for line in out.stdout.splitlines() if line.strip()]
+
+
+# fmt: off
[email protected](_running_as_root(), reason = "root can open a mode 000 node, nothing is shut")
+ pytest.param((None, True, True), id = "a_hidden_vendor_over_an_amd_topology"),
+ pytest.param((None, False, False), id = "a_hidden_vendor_over_no_amd_topology"),
+ pytest.param(("0x10de", True, False), id = "a_readable_foreign_vendor"),
+ pytest.param(("0x1002", False, True), id = "a_readable_amd_vendor"),
+])
+# fmt: on
+def test_which_nodes_the_installer_enumerates_as_closed(tmp_path, case):
+ """The installer half of the closed-node walk, over a real mode 000 file.
+
+ A container can map /dev/dri and hide the sysfs attribute naming its vendor, the shape
+ #10466 is about; dropping the node left the installer printing no render-node repair at
+ all, while _amd_render_node_present reads the same unknown as PRESENT and withdraws the
+ missing-node sentence, so that host got no diagnosis. The vendor guard exists because
+ render nodes are root:render for EVERY vendor, so an NVIDIA-only box has the same closed
+ list and none of the problem. A readable non-AMD vendor stays excluded however the
+ topology reads, since positive evidence beats the fallback and a mixed box must not be
+ told to chgrp its NVIDIA node; a readable AMD one is kept, which is what the enumeration
+ is for.
+
+ Fails before the fix, which required a readable vendor. The Python half has answered this
+ since round twenty-five; this is the installer catching up."""
+ vendor, topology, kept = case
+ node = _a_closed_node_file(tmp_path, "renderD128")
+ closed = _install_sh_closed_nodes(
+ [node], vendors = {node: vendor} if vendor else {}, topology = topology
+ )
+ assert closed == ([str(node)] if kept else [])
+
+
+def _icd_manifest_with(
+ tmp_path,
+ name,
+ *,
+ library = None,
+ arch = None,
+ elf = None,
+):
+ """An ICD manifest with a declared architecture, an ELF library, or neither."""
+ icd = {"api_version": "1.3.0"}
+ if library is not None:
+ lib = tmp_path / library
+ if elf is not None:
+ # e_ident: magic, then EI_CLASS 1 for 32-bit and 2 for 64-bit.
+ lib.write_bytes(b"\x7fELF" + bytes([1 if elf == 32 else 2]) + b"\x00" * 11)
+ else:
+ lib.write_bytes(b"")
+ icd["library_path"] = str(lib)
+ else:
+ icd["library_path"] = "libvulkan_radeon.so"
+ if arch is not None:
+ icd["library_arch"] = arch
+ path = tmp_path / name
+ path.write_text(json.dumps({"file_format_version": "1.0.1", "ICD": icd}), encoding = "utf-8")
+ return str(path)
+
+
+# fmt: off
+ pytest.param(("radeon_icd.json", {"library": "a.so", "arch": "32"}, True),
+ id = "a_declared_32"),
+ pytest.param(("radeon_icd.i686.json", {"library": "b.so", "arch": "64"}, False),
+ id = "a_declared_64_under_an_i686_name"),
+ pytest.param(("radeon_icd.json", {"library": "c.so", "elf": 32}, True), id = "a_32_bit_elf"),
+ pytest.param(("radeon_icd.json", {"library": "d.so", "elf": 64}, False), id = "a_64_bit_elf"),
+ pytest.param(("radeon_icd.i686.json", {}, True), id = "the_filename_alone"),
+])
+# fmt: on
+def test_what_decides_an_icd_manifests_bitness(monkeypatch, linux, tmp_path, case):
+ """library_arch is the loader's own field, read for exactly this purpose: to skip a driver
+ whose bitness cannot match the process. A neutrally named 32-bit registration passed the
+ filename test and credited a driver this binary cannot open, and a manifest named i686
+ that declares 64 is loadable, because the declaration is the loader's answer and the name
+ is only ever a guess at it. The field is optional -- Debian strips it back out of Mesa's
+ manifests to keep one file across architectures -- so its absence is ordinary and the
+ object itself still says: EI_CLASS is byte 4 of every ELF. The filename is the last
+ resort, unchanged, and is also all the installer ever has.
+
+ Fails before the fix, which asked the filename alone."""
+ name, manifest_kwargs, is_32 = case
+ manifest = _icd_manifest_with(tmp_path, name, **manifest_kwargs)
+ # Empty for every arm, so the filename fallback is reached only where there is no
+ # declaration and no library on disk to read.
+ monkeypatch.setattr(amd, "_dynamic_loader_search_dirs", lambda: [])
+ assert amd._an_icd_is_32_bit(manifest) is is_32
+
+
+def test_a_declared_32_bit_manifest_reaches_the_empty_probe_reason(monkeypatch, linux, tmp_path):
+ """What the classification decides: with the only other registration 32-bit, that vendor's
+ open render node is not a path this binary has, so the closed AMD node stays the answer
+ rather than being demoted."""
+ _theirs = _icd_manifest_with(tmp_path, "nvidia_icd.json", library = "libGLX_nvidia.so", arch = "32")
+ _ours = _icd_manifest(tmp_path, "radeon_icd.x86_64.json")
+ _assert_amd_only_loader(
+ _vulkan_reason_under_icd_list(monkeypatch, os.pathsep.join([_theirs, _ours])), True
+ )
+
+
+# fmt: off
+ pytest.param(("DOMAIN\\ada", "render", "usermod -a -G render 'DOMAIN\\ada'", False),
+ id = "an_account_name_the_shell_would_mangle"),
+ pytest.param(("ada", "gpu users", "usermod -a -G 'gpu users' ada", False),
+ id = "a_group_name_that_carries_a_space"),
+ pytest.param(("ada", "render", "usermod -a -G render ada", True), id = "ordinary_names"),
+])
+# fmt: on
+def test_the_pasted_command_quotes_the_names_that_need_it(monkeypatch, linux, case):
+ """These are commands to paste. NSS names are not identifiers -- winbind hands back
+ DOMAIN\\user -- so an unquoted one is de-escaped by the shell and usermod then names an
+ account that does not exist, leaving the node shut; a group name carrying a space is the
+ other half of the same command, and the one that would silently split into two arguments
+ rather than failing outright. shlex.quote rather than unconditional quoting, so the
+ command a user sees on a normal host does not grow quotes it does not need.
+
+ Fails before the fix, which interpolated the name raw."""
+ account, group, command, unquoted = case
+ _nodes(monkeypatch, present = ["/dev/kfd"], openable = set())
+ monkeypatch.setattr(amd, "_repair_account", lambda: account)
+ _owning(monkeypatch, joinable = [group])
+ hint = amd.amd_node_permission_hint()
+ assert command in hint
+ if unquoted:
+ assert "'" not in hint
+
+
+def test_the_installer_quotes_the_account_the_same_way(tmp_path):
+ """The shell twin: the installer prints the same command from the same kind of name, so a
+ host whose account carries a backslash must not get an unquoted one there either."""
+ node, _group = _a_node_a_membership_would_open(tmp_path)
+ out = _install_sh_hint(str(node), id_user = "DOMAIN\\ada", env_user = "DOMAIN\\ada")
+ assert "'DOMAIN\\ada'" in out
+
+
+def test_the_two_quoting_rules_are_the_same_rule(tmp_path):
+ """Both halves print the same command, so a value one quotes and the other does not is a
+ host where the two disagree about what the user should paste. Run against the real shell
+ function rather than a restatement of it."""
+ lines = _install_sh_lines()
+ helper = _shell_fn(lines, "_shell_quote")
+ _values = ("ada", "render", "DOMAIN\\ada", "gpu users", "ada;reboot", "a'b", "user@host", "")
+ for value in _values:
+ out = subprocess.run(
+ ["bash", "-c", helper + '\n_shell_quote "$1"', "_", value],
+ capture_output = True,
+ text = True,
+ )
+ assert out.returncode == 0, out.stderr
+ # What actually matters, asked of the shell rather than of the spelling: each
+ # quoting has to be ONE word that comes back as the name it started as.
+ for _quoted in (out.stdout, shlex.quote(value)):
+ _back = subprocess.run(
+ ["bash", "-c", "printf %s " + _quoted], capture_output = True, text = True
+ )
+ assert _back.returncode == 0, _back.stderr
+ assert _back.stdout == value, (value, _quoted, _back.stdout)
+ # And the printed text is identical too, so the two halves show the same command.
+ # An embedded quote is the one place the spellings differ ('"'"' against \\''), and
+ # both are correct, so that case is carried by the round trip above alone.
+ if "'" not in value:
+ assert out.stdout == shlex.quote(value), (value, out.stdout)
+
+
+# The two helpers every tests/sh ROCm harness lifts alongside _has_amd_rocm_gpu.
+_ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT = frozenset(
+ {"_ensure_rocm_probe_env", "_has_usable_nvidia_gpu"}
+)
+
+
+def _functions_called_by(lines: "list[str]", name: str) -> set:
+ """Which other functions defined in ``lines`` the named one calls."""
+ defined = {line.split("(")[0] for line in lines if re.match(r"^_?[A-Za-z0-9_]+\(\) \{", line)}
+ body = _shell_fn(lines, name).splitlines()[1:]
+ return {
+ _other
+ for _other in defined
+ if _other != name
+ and any(re.search(rf"(^|[\s;&|(]){re.escape(_other)}($|[\s;&|)])", l) for l in body)
+ }
+
+
+def test_the_rocm_probe_calls_nothing_the_shell_harnesses_do_not_lift():
+ """tests/sh lifts probes out of install.sh one function at a time, by name, with
+ `sed -n '/^_name()/,/^}/p'`. So _has_amd_rocm_gpu may only call helpers those harnesses
+ already lift: a call to anything else is an undefined function there, the ROCm branch
+ falls through to the CPU wheel index, and four harnesses fail on a torch-index assertion
+ that has nothing to do with what changed. That is not hypothetical -- splitting the probe
+ into a wrapper over a private _amd_rocm_gpu_visible did exactly this, and the NVIDIA veto
+ now lives inside the function so there is nothing to forget. Fails if the split comes
+ back."""
+ called = _functions_called_by(_install_sh_lines(), "_has_amd_rocm_gpu")
+ assert called <= _ROCM_PROBE_CALLEES_THE_SH_HARNESSES_LIFT, called
+
+
+def test_that_check_sees_a_helper_the_harnesses_would_not_have():
+ """The control. Without it the test above could be passing because the scan matches
+ nothing at all, which is what a name-based scan usually does when it is wrong."""
+ lines = (
+ "_ensure_rocm_probe_env() {\n :\n}\n"
+ "_amd_rocm_gpu_visible() {\n return 1\n}\n"
+ "_has_amd_rocm_gpu() {\n _ensure_rocm_probe_env\n _amd_rocm_gpu_visible\n}"
+ ).splitlines()
+ assert _functions_called_by(lines, "_has_amd_rocm_gpu") == {
+ "_ensure_rocm_probe_env",
+ "_amd_rocm_gpu_visible",
+ }
+
+
+def _multilib_soname(tmp_path, *, bitnesses):
+ """Two loader search directories carrying one soname, and the manifest naming it.
+
+ Named the way a Debian multilib host names them, because the ORDER is the subject:
+ sorted(glob) puts i386-linux-gnu ahead of x86_64-linux-gnu, so a search that stops at
+ the first hit reads the wrong copy on the commonest multilib layout there is.
+ """
+ dirs = []
+ for _arch, _bits in (("i386-linux-gnu", 32), ("x86_64-linux-gnu", 64)):
+ _dir = tmp_path / _arch
+ _dir.mkdir()
+ dirs.append(str(_dir))
+ if _bits in bitnesses:
+ (_dir / "libvk.so").write_bytes(
+ b"\x7fELF" + bytes([1 if _bits == 32 else 2]) + b"\x00" * 11
+ )
+ path = tmp_path / "nvidia_icd.json"
+ path.write_text(
+ json.dumps(
+ {
+ "file_format_version": "1.0.0",
+ "ICD": {"library_path": "libvk.so", "api_version": "1.3.0"},
+ }
+ ),
+ encoding = "utf-8",
+ )
+ return str(path), dirs
+
+
+# fmt: off
+ pytest.param(({32, 64}, 1, False), id = "both_bitnesses_installed"),
+ pytest.param(({32}, 0, True), id = "only_the_wrong_bitness_installed"),
+])
+# fmt: on
+def test_which_copy_of_a_multilib_soname_answers(monkeypatch, tmp_path, case):
+ """Both bitnesses of one soname is what a multilib driver install looks like. ld.so picks
+ the copy matching the process; the reconstructed search order does not, and i386 sorts
+ first, so the first hit was the 32-bit one -- _an_icd_is_32_bit then read that object and
+ discarded a manifest whose driver the loader loads, which either withholds the whole
+ no-driver repair or reports the loader as AMD-only when it is not. The control keeps the
+ rule from becoming "never 32-bit", which would put back every unloadable 32-bit
+ registration the bitness filter exists to drop.
+
+ Fails before the fix, which returned the first match."""
+ bitnesses, chosen, is_32 = case
+ manifest, dirs = _multilib_soname(tmp_path, bitnesses = bitnesses)
+ monkeypatch.setattr(amd, "_dynamic_loader_search_dirs", lambda: dirs)
+ assert amd._icd_library_path(manifest).startswith(dirs[chosen])
+ assert amd._an_icd_is_32_bit(manifest) is is_32
+
+
+def _manifest_missing(tmp_path, name, *, drop):
+ """A manifest with one loader-required field removed, and its library on disk."""
+ lib = tmp_path / f"{name}.so"
+ lib.write_bytes(b"\x7fELF\x02" + b"\x00" * 11)
+ icd = {"library_path": str(lib), "api_version": "1.3.0"}
+ body = {"file_format_version": "1.0.0", "ICD": icd}
+ if drop in icd:
+ del icd[drop]
+ else:
+ del body[drop]
+ path = tmp_path / f"{name}.json"
+ path.write_text(json.dumps(body), encoding = "utf-8")
+ return str(path)
+
+
[email protected]("field", ["file_format_version", "api_version"])
+def test_a_manifest_the_loader_skips_is_not_a_driver(tmp_path, field):
+ """loader_parse_icd_manifest returns VK_ERROR_INCOMPATIBLE_DRIVER on a missing
+ file_format_version and on a missing api_version, exactly as it does on a missing
+ library_path. Reading library_path alone counted a registration the loader refuses, which
+ suppresses the no-driver repair or calls the loader AMD-only when it is neither. Fails
+ before the fix for both fields."""
+ assert amd._icd_manifest_is_usable(_manifest_missing(tmp_path, field, drop = field)) is False
+
+
+def test_a_version_the_loader_does_not_recognise_is_still_a_driver(tmp_path):
+ """The control, and the line between the two. An unknown file_format_version major is the
+ one thing here the loader does NOT skip for: it logs "may cause errors" and carries on, so
+ refusing it would drop a driver that loads. Only absence decides."""
+ lib = tmp_path / "libvk.so"
+ lib.write_bytes(b"\x7fELF\x02" + b"\x00" * 11)
+ path = tmp_path / "future_icd.json"
+ path.write_text(
+ json.dumps(
+ {
+ "file_format_version": "9.0.0",
+ "ICD": {"library_path": str(lib), "api_version": "1.3.0"},
+ }
+ ),
+ encoding = "utf-8",
+ )
+ assert amd._icd_manifest_is_usable(str(path)) is True
+
+
+def _ldconfig_answering(monkeypatch, *, returncode: int, stdout: str) -> None:
+ """A host whose only ldconfig answers exactly this, with the cache read state reset."""
+ monkeypatch.setattr(amd, "_ld_cache_read", False)
+ monkeypatch.setattr(amd, "_ld_cache_sonames_cached", None)
+ monkeypatch.setattr(amd.shutil, "which", lambda _name: "/sbin/ldconfig")
+ monkeypatch.setattr(amd.os.path, "exists", lambda _p: True)
+ monkeypatch.setattr(
+ amd.subprocess,
+ "run",
+ lambda *a, **k: subprocess.CompletedProcess(a[0] if a else [], returncode, stdout, ""),
+ )
+
+
+_EMPTY_CACHE = "0 libs found in cache `/etc/ld.so.cache'\n"
+
+
+# fmt: off
+ pytest.param((0, _EMPTY_CACHE, frozenset()), id = "a_readable_but_empty_cache"),
+ pytest.param((1, "", None), id = "an_ldconfig_that_fails"),
+])
+# fmt: on
+def test_what_an_ldconfig_answer_means(monkeypatch, linux, case):
+ """glibc separates the two states by EXIT STATUS, not by row count: a cache that is
+ present and empty prints "0 libs found in cache" and exits 0, while an absent cache file
+ exits 1 with nothing on stdout. Storing only a non-empty set collapsed them, so a fresh
+ container whose cache has not been built read as "cannot enumerate". The non-zero exit is
+ the absent-cache case and must stay None, so a live driver is never called stale."""
+ returncode, stdout, sonames = case
+ _ldconfig_answering(monkeypatch, returncode = returncode, stdout = stdout)
+ if sonames is None:
+ assert amd._ld_cache_sonames() is None
+ else:
+ assert amd._ld_cache_sonames() == sonames
+
+
+def test_a_bare_soname_is_stale_when_the_readable_cache_does_not_carry_it(
+ monkeypatch, linux, tmp_path
+):
+ """What the distinction is for. The soname is on no directory ld.so searches and in a
+ cache that could be read, so it does not resolve -- and the manifest naming it is a
+ registration with no driver behind it. Read as unknown, it answered usable, which is the
+ arm that withholds the reinstall half of the repair."""
+ manifest = _bare_soname_manifest(tmp_path, "radeon_icd.json", "libvulkan_radeon.so")
+ monkeypatch.setattr(amd, "_dynamic_loader_search_dirs", lambda: [str(tmp_path / "lib")])
+ _ldconfig_answering(monkeypatch, returncode = 0, stdout = _EMPTY_CACHE)
+ assert amd._icd_manifest_is_usable(manifest) is False
+
+
+def _kernel_stack_hint_block() -> str:
+ """The whole diagnosis chain, from its `if` through the closing `fi`."""
+ lines = _install_sh_lines()
+ end = _install_sh_anchor(lines, _PCI_SENTENCE)
+ start = _install_sh_if_above(lines, end)
+ close = next(i for i in range(end + 1, len(lines)) if lines[i] == "fi")
+ return "\n".join(lines[start : close + 1])
+
+
+def test_the_kernel_stack_advice_is_gated_on_the_node_being_absent():
+ """The chain's guard is that /dev/kfd is not CLOSED, which is equally true when the node is
+ absent and when it is open, so an openable /dev/kfd reached a sentence saying there was
+ none -- and /dev/kfd IS the amdkfd char device, so its presence proves the stack is loaded
+ and a reinstall repairs nothing. Read off install.sh because a host with the node cannot
+ be fabricated here: the test operator is live by design, and the arm below skips without
+ one. A revert removes the `[ -e /dev/kfd ]` arm and this raises rather than passing
+ quietly."""
+ block = _kernel_stack_hint_block()
+ # Both arms now carry the topology gate as well, since a node that merely EXISTS does
+ # not establish that the AMD driver is what created it.
+ present = block.index("[ -e /dev/kfd ] && _kfd_node_is_amds")
+ absent = block.index("[ ! -e /dev/kfd ] || ! _kfd_node_is_amds")
+ assert present < block.index("kernel stack is already loaded") < absent
+ assert absent < block.index("Install the ROCm kernel stack")
+
+
+def test_the_installer_names_the_userspace_when_the_node_is_already_there(tmp_path):
+ """The executed half, for a host that has the node. It used to skip unless the RUNNER
+ had /dev/kfd, which is the one machine shape a dev box never is; the arm it pairs with
+ skipped on exactly the complement, so the pair was never both run anywhere. The `-e`
+ operator is still executed -- only the path it is given belongs to this case."""
+ out = _kernel_stack_hint_text(tmp_path, topology = True, kfd_present = True)
+ assert "Install the ROCm kernel stack" not in out
+ assert "kernel stack is already loaded" in out
+ assert "rocminfo" in out
+ # Codex 4050704658, and the reason the arm above now needs a topology naming AMD: a
+ # /dev/kfd that exists proves a node exists, not that the AMD driver created it. Where
+ # the topology is READ and holds no AMD agent, the kernel stack is not loaded and the
+ # userspace-only advice is the wrong repair.
+ out = _kernel_stack_hint_text(tmp_path, topology = False, kfd_present = True)
+ assert "kernel stack is already loaded" not in out
+ assert "Install the ROCm kernel stack" in out
+ # ...but a topology that cannot be READ is unproven rather than contradicted, so the
+ # container masking /sys/class/kfd keeps the advice it has today.
+ out = _kernel_stack_hint_text(
+ tmp_path, topology = False, topology_readable = False, kfd_present = True
+ )
+ assert "kernel stack is already loaded" in out
+
+
+def _backend_env(**env: str) -> dict:
+ """The runner's environment with both backend requests cleared, plus this case's."""
+ _base = {
+ k: v
+ for k, v in os.environ.items()
+ if k not in ("UNSLOTH_LLAMA_CPP_BACKEND", "UNSLOTH_FORCE_VULKAN")
+ }
+ return {**_base, **env}
+
+
+def _resolved_backend(**env: str) -> str:
+ """What install.sh resolves the backend request to, for one environment."""
+ script = "\n".join(
+ [
+ _shell_fn(_install_sh_lines(), "_requested_llama_backend"),
+ "_requested_llama_backend",
+ ]
+ )
+ return _install_sh_run(script, env = _backend_env(**env)).strip()
+
+
+def _gpu_node_scope(**env: str) -> str:
+ """Whether _run_may_open_a_gpu_node fires, on an NVIDIA host with --no-torch.
+
+ That host is the one the automatic route answers NO for, so anything that says yes here
+ says it because the run named a backend.
+ """
+ lines = _install_sh_lines()
+ script = "\n".join(
+ [
+ "SKIP_TORCH=true",
+ "TORCH_INDEX_URL=''",
+ "_has_usable_nvidia_gpu() { return 0; }",
+ _shell_fn(lines, "_requested_llama_backend"),
+ _shell_fn(lines, "_torch_index_url_leaf"),
+ _shell_fn(lines, "_is_pip_rocm_family_leaf"),
+ _shell_fn(lines, "_torch_opens_amd_nodes"),
+ _shell_fn(lines, "_auto_bundle_opens_amd_nodes"),
+ _shell_fn(lines, "_run_may_open_a_gpu_node"),
+ "_run_may_open_a_gpu_node && echo yes || echo no",
+ ]
+ )
+ return _install_sh_run(script, env = _backend_env(**env)).strip()
+
+
+_FORCE_VULKAN = "UNSLOTH_FORCE_VULKAN"
+_BACKEND = "UNSLOTH_LLAMA_CPP_BACKEND"
+
+
+# fmt: off
+ pytest.param(({_FORCE_VULKAN: "1"}, "vulkan", "yes"), id = "the_legacy_flag_alone"),
+ pytest.param(({_BACKEND: "cuda", _FORCE_VULKAN: "1"}, "cuda", "no"),
+ id = "cuda_over_the_legacy_flag"),
+ pytest.param(({_BACKEND: "auto", _FORCE_VULKAN: "1"}, "auto", "no"),
+ id = "auto_over_the_legacy_flag"),
+ pytest.param(({_FORCE_VULKAN: "0"}, "", "no"), id = "the_legacy_flag_set_to_zero"),
+])
+# fmt: on
+def test_how_the_legacy_vulkan_flag_is_resolved(case):
+ """effective_backend_request resolves UNSLOTH_FORCE_VULKAN through
+ environment_backend_override, so a run that sets only the legacy flag installs the Vulkan
+ bundle; reading the new variable alone left that run on the automatic route here, where an
+ NVIDIA card answers no and every render-node diagnosis is suppressed for an install that
+ opens exactly those nodes. A recognised public value is authoritative, so the legacy
+ boolean cannot pull a cuda install back to Vulkan -- and "auto" is such a value, a request
+ to DETECT, which is why the override returns it rather than falling through; otherwise the
+ fix reads as "the legacy flag always wins" and takes a host that asked for detection off
+ the automatic route. Only the four truthy spellings count, exactly as the Python side
+ lists them, so a 0 leaves the automatic route alone."""
+ env, resolved, scope = case
+ assert _resolved_backend(**env) == resolved
+ assert _gpu_node_scope(**env) == scope
+
+
+_VK_MASK = "GGML_VK_VISIBLE_DEVICES"
+
+
+# fmt: off
+ pytest.param(({_VK_MASK: ""}, {"vulkan"},
+ ("visibility mask is also in force", f"{_VK_MASK} is empty"), ()),
+ id = "an_empty_mask_on_a_vulkan_build"),
+ pytest.param(({_VK_MASK: "3"}, {"vulkan"},
+ ("names a device this cannot resolve", f"{_VK_MASK}='3'"), ()),
+ id = "an_unboundable_mask_on_a_vulkan_build"),
+ pytest.param(({_VK_MASK: ""}, {"hip"}, (), (_VK_MASK,)), id = "the_same_mask_on_a_hip_build"),
+ pytest.param(({}, {"vulkan"}, (), (_VK_MASK,)), id = "no_such_mask_at_all"),
+ # A negative ordinal is the one out-of-range value decidable WITHOUT the raw device
+ # count: ggml extracts with `size_t tmp; while (ss >> tmp)`, unsigned extraction runs
+ # strtoull, and "-1" wraps to 2**64-1, which is >= any possible num_available_devices.
+ # So it always throws "Invalid Vulkan device index" and no group membership repairs it.
+ # Reported as a blocker, never as a mask merely worth checking after the group repair.
+ pytest.param(({_VK_MASK: "-1"}, {"vulkan"},
+ ("visibility mask is also in force", f"{_VK_MASK}='-1'"),
+ ("names a device this cannot resolve",)),
+ id = "a_negative_ordinal_always_throws_so_it_blocks"),
+ # Extraction WALKS the list, so a negative one throws wherever it sits, as long as
+ # every token ahead of it still extracts.
+ pytest.param(({_VK_MASK: "0,-1"}, {"vulkan"},
+ ("visibility mask is also in force",),
+ ("names a device this cannot resolve",)),
+ id = "a_negative_ordinal_behind_a_valid_one_still_blocks"),
+ # ... but only as far as the first token that does NOT extract: ggml stops reading
+ # there, so the negative one is never reached and cannot be what throws.
+ pytest.param(({_VK_MASK: "abc,-1"}, {"vulkan"},
+ ("visibility mask is also in force",), ()),
+ id = "a_negative_ordinal_after_a_dead_token_is_never_read"),
+ # "-0" wraps to 0, which is in range on any host with a device, so it is not a throw
+ # and must not be promoted to a blocker: that would invent the fault this guards.
+ pytest.param(({_VK_MASK: "-0"}, {"vulkan"},
+ ("names a device this cannot resolve", f"{_VK_MASK}='-0'"), ()),
+ id = "a_negative_zero_is_in_range_and_stays_unresolved"),
+])
+# fmt: on
+def test_when_the_vulkan_selector_is_named_beside_the_node(monkeypatch, linux, case):
+ """A Vulkan build reads none of the four HIP/CUDA selectors, so the loop above skips them
+ all and the node repair was returned as the complete explanation. ggml reads
+ GGML_VK_VISIBLE_DEVICES itself and _run_vulkan_probe passes it through, and an empty value
+ extracts no ordinal at all, so the probe stays empty however the node is owned. Its
+ ordinals index the RAW vkEnumeratePhysicalDevices list, before CPU devices are dropped and
+ ICDs deduplicated, so this process does not have the bound and a merely large positive
+ ordinal throws rather than hiding: reported to check rather than called a blocker, since
+ naming it one would invent a fault. A NEGATIVE one is the exception, decidable without the
+ bound because it wraps past every possible one. A build reads its own selectors and no
+ others, so naming this
+ one to a HIP install sends the user after a variable its runtime never reads; and unset is
+ not empty, so the rule cannot be "always mention it"."""
+ env, backends, present, absent = case
+ reason = _reason_with_masks(monkeypatch, env, backends)
+ for _text in present:
+ assert _text in reason, _text
+ for _text in absent:
+ assert _text not in reason, _text
+
+
+def test_the_two_topology_readers_agree_on_this_host():
+ """The shell and Python tri-states are the same rule written twice, and the closed-node
+ walk on each side now branches on the third value, so a divergence would give one half the
+ DRM fallback and not the other. Asserted as AGREEMENT rather than as a fixed value, so the
+ arm is meaningful on a runner with a real KFD topology as well as on one without."""
+ lines = _install_sh_lines()
+ script = "\n".join(
+ [
+ _shell_fn(lines, "_kfd_topology_amd_state"),
+ "_st=0",
+ "_kfd_topology_amd_state || _st=$?",
+ 'printf "%s" "$_st"',
+ ]
+ )
+ shell_state = int(_install_sh_run(script).strip())
+ python_state = amd._kfd_topology_amd_state()
+ assert shell_state == {True: 0, False: 1, None: 2}[python_state]
+
+
+def test_the_two_confirmed_render_node_readers_agree_on_this_host():
+ """Its companion, and the other half of the fallback: the shell one has to be as strict as
+ the Python one, or an NVIDIA-only host claims an AMD node in the installer and not in the
+ backend."""
+ lines = _install_sh_lines()
+ script = "\n".join(
+ [
+ _shell_fn(lines, "_amd_render_node_vendor"),
+ _shell_fn(lines, "_a_confirmed_amd_render_node_exists"),
+ "_a_confirmed_amd_render_node_exists && printf yes || printf no",
+ ]
+ )
+ assert _install_sh_run(script).strip() == (
+ "yes" if amd._a_confirmed_amd_render_node_exists() else "no"
+ )
+
+
+def test_the_installer_kfd_arm_consults_the_same_fallback():
+ """Read off install.sh because a /dev/kfd cannot be fabricated here: the arm is gated on
+ the literal path and on live -e/-r/-w tests, which are the rule under test. Without the
+ fallback the installer names only the render node's group, and where the two nodes carry
+ different owning groups that membership leaves KFD shut -- and the PCI branch further down
+ then reads the node as openable. A revert removes these names and this fails rather than
+ passing quietly."""
+ lines = _install_sh_lines()
+ body = _shell_fn(lines, "_amd_nodes_closed_to_this_user")
+ _kfd = body.index("= /dev/kfd ]")
+ _elif = body.index("elif _node_vendor=")
+ _arm = body[_kfd:_elif]
+ assert "_kfd_topology_amd_state" in _arm
+ assert "_a_confirmed_amd_render_node_exists" in _arm
+ # The readable-but-not-AMD state still drops the node, which is what keeps an
+ # NVIDIA-only host silent; only the unreadable one reaches DRM.
+ assert "-eq 1 ]; then" in _arm and "continue" in _arm
+
+
+def _loader_blame(
+ monkeypatch,
+ manifests: dict,
+ searched = (),
+ **env: str,
+) -> "str | None":
+ """Which override amd.py blames for a loader that can load none of its manifests.
+
+ The dict value says whether that manifest still resolves to a library, which is the
+ difference between "clear the variable" and "reinstall the driver": these paths do not
+ exist on the test host, so the real _icd_manifest_is_usable would call every one of them
+ broken. ``searched`` is what the ordinary search would find with a forced list cleared,
+ which is the only way a forced list can be the repair.
+ """
+ for var in _VK_OVERRIDE_VARS:
+ monkeypatch.delenv(var, raising = False)
+ for var, value in env.items():
+ monkeypatch.setenv(var, value)
+ usable = dict(manifests)
+ usable.update({path: True for path in searched})
+ monkeypatch.setattr(amd, "_vulkan_icd_manifest_paths", lambda: list(manifests))
+ monkeypatch.setattr(amd, "_searched_vulkan_icd_manifest_paths", lambda: list(searched))
+ monkeypatch.setattr(amd, "_icd_manifest_is_usable", lambda path: bool(usable.get(path)))
+ monkeypatch.setattr(amd, "_an_icd_is_32_bit", lambda _path: False)
+ return amd.the_vulkan_loader_override_to_blame()
+
+
+_SELECT = "VK_LOADER_DRIVERS_SELECT"
+_DISABLE = "VK_LOADER_DRIVERS_DISABLE"
+_FORCED = "VK_DRIVER_FILES"
+_ICD_RADEON = "/etc/vulkan/icd.d/radeon_icd.x86_64.json"
+_ICD_SEARCHED = "/etc/vulkan/icd.d/radeon.json"
+_ICD_GONE = "/gone/radeon.json"
+
+
+# fmt: off
+ pytest.param(({_ICD_RADEON: True}, (), {_DISABLE: "*"}, _DISABLE), id = "a_disable_glob"),
+ pytest.param(({_ICD_RADEON: True}, (), {_SELECT: "nvidia*"}, _SELECT), id = "a_select_list"),
+ pytest.param(({_ICD_GONE: False}, (_ICD_SEARCHED,), {_FORCED: _ICD_GONE}, _FORCED),
+ id = "a_forced_list_pointing_at_nothing"),
+ pytest.param(({_ICD_RADEON: True}, (), {}, None), id = "no_override_at_all"),
+ pytest.param(({_ICD_RADEON: True}, (), {_SELECT: "radeon*", _DISABLE: "radeon*"}, _DISABLE),
+ id = "select_and_disable_naming_the_same_driver"),
+ pytest.param(({_ICD_RADEON: True}, (), {_SELECT: "nvidia*", _DISABLE: "intel*"}, _SELECT),
+ id = "select_excluding_it_and_disable_missing_it"),
+ pytest.param(({_ICD_RADEON: True}, (), {_SELECT: "nvidia*", _DISABLE: "radeon*"},
+ f"{_SELECT} and {_DISABLE} together"),
+ id = "select_and_disable_each_excluding_it"),
+ pytest.param(({__file__: False}, (), {_FORCED: __file__}, None),
+ id = "a_forced_list_whose_manifest_is_permitted"),
+ pytest.param(({__file__: False}, (_ICD_SEARCHED,), {_FORCED: __file__}, _FORCED),
+ id = "the_same_list_hiding_a_usable_search"),
+ pytest.param(({_ICD_GONE: False}, (_ICD_SEARCHED,), {_FORCED: _ICD_GONE, _DISABLE: "radeon*"},
+ f"{_FORCED} and {_DISABLE} together"),
+ id = "a_stale_forced_path_under_a_filter_that_also_blocks_it"),
+ pytest.param(({_ICD_GONE: False}, (_ICD_SEARCHED,), {_FORCED: _ICD_GONE}, _FORCED),
+ id = "the_forced_list_of_that_pair_on_its_own"),
+ pytest.param(({__file__: True}, (), {_FORCED: __file__, _DISABLE: "*"}, _DISABLE),
+ id = "the_filter_of_that_pair_on_its_own"),
+ pytest.param(({_ICD_RADEON: False}, (), {_DISABLE: "*"}, None),
+ id = "a_filter_over_a_manifest_whose_library_is_gone"),
+ pytest.param(({_ICD_RADEON: True}, (), {_DISABLE: "*"}, _DISABLE),
+ id = "the_same_filter_over_a_loadable_manifest"),
+ pytest.param(({}, (), {_DISABLE: "*"}, None), id = "a_loader_with_no_manifests_at_all"),
+])
+# fmt: on
+def test_which_override_a_driverless_loader_blames(monkeypatch, case):
+ """Which variable clearing would actually restore a driver, and None where none would.
+
+ DISABLE applies to every driver the loader knows, so a list matching them all leaves it
+ with none and no reinstall changes an environment variable; the sentence prescribed
+ exactly that repair. SELECT is read first because _vulkan_loader_allows reads it first: a
+ set select list answers alone, so one naming a driver this host lacks excludes the ones it
+ has. A forced list REPLACES the search, so a stale path leaves the loader with manifests
+ that do not resolve while the real drivers sit unread, and reinstalling puts a driver
+ where nothing is looking. With no override the manifests are what they are and installing
+ a driver IS the repair, so the rule cannot be "always blame the environment".
+
+ Disable WINS over select, so clearing select after both name the same driver changes
+ nothing -- advice that cannot work, and what this answered before. Select is named where
+ select is what excludes the manifest; both are named where neither removal alone repairs,
+ and the same across kinds, since clearing a filter can leave a manifest that is not there
+ while clearing the list exposes a search the filter then empties. The one-sided controls
+ sit beside those, so this is not "always say both". A forced list is only the repair when
+ clearing it leaves a driver: a present, permitted manifest that has merely lost its
+ library is the reinstall case, while the same list hiding a search that WOULD have found
+ one is the repair after all.
+
+ Codex 4040623253: the counterfactual tested filename allowance, not loadability, so a
+ filter over a manifest that no longer resolves was named as the whole repair -- clear the
+ variable, and the loader keeps the same unloadable manifest and no driver. Its control
+ keeps that from becoming "never blame a filter". Finding no manifests at all says only
+ that this cannot read the loader's configuration, which
+ the_vulkan_loader_has_no_usable_driver already answers False for, so there is nothing to
+ name."""
+ manifests, searched, env, blamed = case
+ _answer = _loader_blame(monkeypatch, manifests, searched = searched, **env)
+ if blamed is None:
+ assert _answer is None
+ else:
+ assert _answer == blamed
+
+
+# fmt: off
+ pytest.param((True, {_DISABLE: "*"}, ("no driver it can load", _DISABLE),
+ ("reinstall the Vulkan driver",)),
+ id = "a_filter_over_a_manifest_that_resolves"),
+ pytest.param((False, None, ("reinstall the Vulkan driver",), ()),
+ id = "no_override_and_a_library_that_is_gone"),
+])
+# fmt: on
+def test_which_repair_the_no_driver_sentence_prescribes(monkeypatch, linux, tmp_path, case):
+ """The message a user actually reads, since the helper above only decides it. The manifest
+ in the first case resolves perfectly well and the filter is the whole reason the loader
+ has nothing, so "reinstall the Vulkan driver" is a repair that cannot work; the control is
+ the case the sentence WAS written for, without which the fix could be "never say
+ reinstall"."""
+ library_present, env, present, absent = case
+ _icd_manifest(tmp_path, "radeon_icd.json", present = library_present)
+ reason = _vulkan_node_hint_under_icd_list(
+ monkeypatch, None, search_dirs = [str(tmp_path)], env = env
+ )
+ for _text in present:
+ assert _text in reason, _text
+ for _text in absent:
+ assert _text not in reason, _text
+
+
+def _repair_user_under(id_stub: str) -> str:
+ """What install.sh captures as the account to name, given this `id`.
+
+ The assignment is lifted from the file rather than restated, so reverting it fails this.
+ """
+ lines = _install_sh_lines()
+ i = _install_sh_anchor(lines, "_amd_repair_user=$(id -un")
+ script = "\n".join([id_stub, lines[i].strip(), 'printf "%s" "$_amd_repair_user"'])
+ out = subprocess.run(["sh", "-c", script], capture_output = True, text = True, check = True)
+ return out.stdout
+
+
+# fmt: off
+ pytest.param(("id() { echo 12345; return 1; }", ""), id = "a_uid_with_no_passwd_entry"),
+ pytest.param(("id() { echo ada; return 0; }", "ada"), id = "a_resolvable_account"),
+])
+# fmt: on
+def test_which_account_the_installer_names(case):
+ """`docker run --user 1234`, which is the shape the paragraph above the assignment is
+ about. GNU id PRINTS the uid and THEN exits 1 for a uid it cannot resolve (coreutils id.c,
+ print_user falls back to uidtostr), so a `|| printf ''` fallback never runs and the
+ captured value was the number. The callers read empty as "no account to name" and print
+ the container repair; a numeric one reached `usermod -a -G render 1234`, which usermod
+ rejects with "user '1234' does not exist". Verified against real GNU coreutils in a
+ container before this was written. The control keeps the fix from becoming "never name an
+ account", which removes the usermod prescription on every ordinary host."""
+ id_stub, named = case
+ assert _repair_user_under(id_stub) == named
+
+
+def _topology_state(monkeypatch, tmp_path, entries: "dict[str, str | None]"):
+ """_kfd_topology_amd_state over a fabricated node tree.
+
+ A value of None is a properties file that will not open, which is what a masked sysfs
+ and an LSM both produce; the others are the file's contents. os.listdir and open are
+ redirected on the MODULE rather than on builtins: a global open patch recurses, since
+ pathlib opens files to answer the patch.
+ """
+ real = tmp_path / "nodes"
+ for name, body in entries.items():
+ node = real / name
+ node.mkdir(parents = True)
+ properties = node / "properties"
+ properties.write_text(body or "", encoding = "utf-8")
+ if body is None:
+ properties.chmod(0o000)
+ prefix = "/sys/class/kfd/kfd/topology/nodes"
+
+ def _redirect(path):
+ return str(path).replace(prefix, str(real))
+
+ real_listdir, real_open = os.listdir, open
+ monkeypatch.setattr(amd.os, "listdir", lambda p: real_listdir(_redirect(p)))
+ monkeypatch.setattr(
+ amd, "open", lambda p, *a, **k: real_open(_redirect(p), *a, **k), raising = False
+ )
+ return amd._kfd_topology_amd_state()
+
+
+# fmt: off
[email protected](_running_as_root(), reason = "root opens a 0000 file, so nothing is unreadable")
+ pytest.param(({"0": "cpu_cores_count 16\nsimd_count 0\nvendor_id 0\n", "1": None}, None),
+ id = "a_gpu_node_that_will_not_open"),
+ pytest.param(({"0": "cpu_cores_count 16\nvendor_id 0\n",
+ "1": "simd_count 128\nvendor_id 4318\n"}, False),
+ id = "every_node_read_and_none_amd"),
+ pytest.param(({"0": "cpu_cores_count 16\nvendor_id 0\n", "1": None,
+ "2": "simd_count 256\nvendor_id 4098\n"}, True),
+ id = "an_amd_node_beside_an_unreadable_sibling"),
+])
+# fmt: on
+def test_what_an_unreadable_topology_node_answers(monkeypatch, tmp_path, case):
+ """The CPU node opens and the GPU node does not: one entry short of "names none". False
+ there drops /dev/kfd from the closed list, so a host whose KFD is owned by video and whose
+ render node is owned by render is told to join render alone and left with KFD shut -- the
+ node the ROCm caller actually needs. install.sh states the same rule for the same
+ decision: a topology that could not be READ is not one that named another vendor. The
+ controls keep the fix from becoming "never answer False", which would let an NVIDIA-only
+ host whose KFD nodes all read as 4318 claim an AMD card, and confirm that one AMD node
+ answers True however many siblings failed."""
+ entries, state = case
+ assert _topology_state(monkeypatch, tmp_path, entries) is state
+
+
+ "select,disable,allowed",
+ [
+ ("", "", True),
+ ("radeon*", "", True),
+ ("nvidia*", "", False),
+ ("", "radeon*", False),
+ # The case Khronos settles: disable is considered BEFORE select, and drivers have no
+ # VK_LOADER_LAYERS_ALLOW counterpart to name one back, so the real loader ends up
+ # with no driver here. Read as "select answers alone" this counted Radeon usable and
+ # reported only the device-node repair for a host groups cannot fix.
+ ("radeon*", "radeon*", False),
+ ("radeon*", "nvidia*", True),
+ ("radeon*,nvidia*", "nvidia*", True),
+ ],
+)
+def test_the_loader_filters_are_an_allowlist_then_a_denylist(monkeypatch, select, disable, allowed):
+ """Vulkan-Loader, LoaderInterfaceArchitecture.md: "The values from the disable
+ environment variable will be considered before the enable or select environment
+ variable", and VK_LOADER_DRIVERS_DISABLE is "also checked before other driver
+ environment variables (such as VK_LOADER_DRIVERS_SELECT)"."""
+ monkeypatch.setenv("VK_LOADER_DRIVERS_SELECT", select)
+ monkeypatch.setenv("VK_LOADER_DRIVERS_DISABLE", disable)
+ assert amd._vulkan_loader_allows("/usr/share/vulkan/icd.d/radeon_icd.x86_64.json") is allowed
+
+
+def _lacking(monkeypatch, *, topology, confirmed_drm, kfd_present, render_present):
+ monkeypatch.setattr(amd, "_kfd_topology_amd_state", lambda: topology)
+ monkeypatch.setattr(amd, "_kfd_topology_has_an_amd_gpu", lambda: topology is True)
+ monkeypatch.setattr(amd, "_a_confirmed_amd_render_node_exists", lambda: confirmed_drm)
+ monkeypatch.setattr(amd, "_amd_render_node_exists", lambda: render_present)
+ monkeypatch.setattr(
+ amd.os.path, "exists", lambda p: kfd_present if p == amd._KFD_NODE else os.path.exists(p)
+ )
+ return amd._amd_nodes_the_runtime_lacks(needs_kfd = True)
+
+
+# fmt: off
+ pytest.param((None, True, [amd._KFD_NODE]), id = "a_masked_topology_drm_confirms"),
+ pytest.param((None, False, []), id = "a_masked_topology_drm_cannot_confirm"),
+ pytest.param((False, True, []), id = "a_topology_that_names_no_amd_gpu"),
+])
+# fmt: on
+def test_when_a_missing_kfd_is_reported_as_missing(monkeypatch, case):
+ """`docker run --device /dev/dri` without `--device /dev/kfd`, which commonly masks
+ /sys/class/kfd too. The topology proves nothing there, but DRM names an AMD render node
+ outright, and a HIP caller cannot run without /dev/kfd -- so suppressing the diagnosis left
+ that host with the generic "no GPU" reading of its own missing device mapping. The DRM
+ evidence must be the CONFIRMED kind, since an NVIDIA-only host has render nodes under the
+ same glob and reading an unreadable vendor as AMD would hand it a ROCm device-mapping
+ repair for a card it does not have. And READ and denied is not unknown, so a host whose
+ KFD nodes all report another vendor gets nothing whatever DRM says."""
+ topology, confirmed_drm, lacks = case
+ assert (
+ _lacking(
+ monkeypatch,
+ topology = topology,
+ confirmed_drm = confirmed_drm,
+ kfd_present = False,
+ render_present = True,
+ )
+ == lacks
+ )
+
+
+def test_the_wording_does_not_claim_a_loaded_driver_it_cannot_prove(monkeypatch):
+ """The masked-topology route has not read the amdkfd driver's own sysfs, so it must not
+ say the kernel driver is loaded and a reinstall is pointless. The confirmed-topology route
+ still does, which is the control."""
+ monkeypatch.setattr(amd, "_amd_nodes_the_runtime_lacks", lambda **_k: [amd._KFD_NODE])
+ monkeypatch.setattr(amd, "amd_nodes_closed_to_this_user", lambda **_k: [])
+
+ monkeypatch.setattr(amd, "_kfd_topology_amd_state", lambda: None)
+ masked = amd.amd_node_permission_hint(needs_kfd = True) or ""
+ assert "the kernel driver is loaded" not in masked
+ assert "/dev/kfd" in masked and "--device /dev/kfd" in masked
+
+ monkeypatch.setattr(amd, "_kfd_topology_amd_state", lambda: True)
+ named = amd.amd_node_permission_hint(needs_kfd = True) or ""
+ assert "the kernel driver is loaded" in named
+
+
+# fmt: off
+ pytest.param(([amd._KFD_NODE], ("/dev/kfd",)), id = "with_a_second_finding_after_it"),
+ pytest.param(([], ()), id = "when_the_command_is_the_only_finding"),
+])
+# fmt: on
+def test_the_message_ends_with_the_command_a_user_pastes(monkeypatch, case):
+ """The membership sentence ends in a command a user copies, and nothing may follow it.
+
+ Rendered in place it ran straight into the next sentence: "... -a -G render ada ROCm needs
+ /dev/kfd ...", where the command a reader selects to end-of-line picks up the word after
+ it. Ending the whole message with it is what keeps the command copyable as written; a full
+ stop would be selected along with the account name instead. The sentence that used to
+ follow it is still present, just no longer glued on, and with nothing else to say the hint
+ is still the repair, so nothing is lost by holding it back to the end."""
+ lacks, also_present = case
+ monkeypatch.setattr(amd, "amd_nodes_closed_to_this_user", lambda **_k: ["/dev/dri/renderD128"])
+ monkeypatch.setattr(amd, "_amd_nodes_the_runtime_lacks", lambda **_k: lacks)
+ monkeypatch.setattr(amd, "_kfd_topology_amd_state", lambda: True)
+ monkeypatch.setattr(amd, "_repair_account", lambda: "ada")
+ monkeypatch.setattr(
+ amd, "_groups_that_own", lambda _paths: (["render"], [], [], [], [], [], [], [])
+ )
+ hint = amd.amd_node_permission_hint(needs_kfd = True) or ""
+
+ assert "sudo usermod -a -G render ada" in hint
+ assert hint.rstrip().endswith("sudo usermod -a -G render ada")
+ for _text in also_present:
+ assert _text in hint
diff --git a/studio/backend/tests/test_install_resolve_prebuilt.py b/studio/backend/tests/test_install_resolve_prebuilt.py
index cc1f7f415e5..68c4b806ab5 100644
--- a/studio/backend/tests/test_install_resolve_prebuilt.py
+++ b/studio/backend/tests/test_install_resolve_prebuilt.py
@@ -2199,9 +2199,16 @@ def test_a_manifest_the_loader_filters_out_does_not_answer_for_the_driver(monkey
monkeypatch.setenv("VK_LOADER_DRIVERS_DISABLE", "*radeon*")
assert ilp._amd_vulkan_icd_present() is False
- # Disable is read first so a select list names drivers back in, which is the loader's order.
+ # Disable is read first and WINS: "the values from the disable environment variable will
+ # be considered before the enable or select environment variable" (Vulkan-Loader,
+ # LoaderInterfaceArchitecture.md), and drivers have no VK_LOADER_LAYERS_ALLOW counterpart
+ # to name one back in. This asserted the opposite, which is the misreading that counted a
+ # disabled Radeon as usable.
monkeypatch.setenv("VK_LOADER_DRIVERS_SELECT", "RADEON_ICD.X86_64.JSON")
+ assert ilp._amd_vulkan_icd_present() is False
+ monkeypatch.delenv("VK_LOADER_DRIVERS_DISABLE")
assert ilp._amd_vulkan_icd_present() is True
+ monkeypatch.setenv("VK_LOADER_DRIVERS_DISABLE", "*radeon*")
# And a select list naming someone else's driver excludes this one on its own.
monkeypatch.setenv("VK_LOADER_DRIVERS_SELECT", "intel_*")
assert ilp._amd_vulkan_icd_present() is False
diff --git a/studio/backend/utils/hardware/amd.py b/studio/backend/utils/hardware/amd.py
index cd3c724690b..716d6c24786 100644
--- a/studio/backend/utils/hardware/amd.py
+++ b/studio/backend/utils/hardware/amd.py
@@ -3,14 +3,18 @@
"""AMD GPU monitoring via amd-smi. Mirrors nvidia.py so hardware.py can swap backends based on IS_ROCM; all functions return the same dict shapes as their nvidia.py counterparts."""
+import glob
import json
import math
import os
import platform
import re
+import shlex
import shutil
+import stat
import subprocess
import sys
+from pathlib import PurePath
from typing import Any, Optional
from loggers import get_logger
@@ -483,3 +487,1403 @@ def get_visible_gpu_utilization(
"devices": devices,
"index_kind": "physical",
}
+
+
+# /dev/kfd is what HIP opens; /dev/dri/renderD* is what BOTH HIP and the Vulkan loader
+# open, so switching backend is no way around a closed render node.
+_KFD_NODE = "/dev/kfd"
+_DRI_RENDER_GLOB = "/dev/dri/renderD*"
+
+
+_AMD_PCI_VENDOR_ID = "0x1002"
+
+
+def _render_node_vendor(path: str) -> "str | None":
+ """The PCI vendor id behind a ``/dev/dri/renderD*`` node, or None if it cannot be read.
+
+ None is not "some other vendor": a container can map the node while masking or not
+ mounting the sysfs entry that names it, and the callers answer differently for the two.
+ """
+ vendor_file = f"/sys/class/drm/{os.path.basename(path)}/device/vendor"
+ try:
+ with open(vendor_file, encoding = "utf-8") as fh:
+ return fh.read().strip().lower()
+ except (OSError, UnicodeDecodeError):
+ return None
+
+
+def _render_node_is_amd(path: str) -> bool:
+ """Whether a ``/dev/dri/renderD*`` node belongs to an AMD GPU.
+
+ Render nodes are ``root:render`` for EVERY vendor, so an NVIDIA-only host has
+ exactly the same closed nodes and none of the problem -- CUDA opens
+ ``/dev/nvidia*`` instead. Without this, the render-group advice below would be
+ given to every CUDA user whose probe came back empty for an unrelated reason.
+ Read from sysfs, which is world-readable, so the answer does not need the access
+ this is testing for.
+ """
+ return _render_node_vendor(path) == _AMD_PCI_VENDOR_ID
+
+
+def _kfd_topology_amd_state() -> "bool | None":
+ """Whether KFD's topology names an AMD GPU, or ``None`` when it cannot be read.
+
+ _kfd_topology_has_an_amd_gpu collapses those two, which is right wherever the question
+ is "is there evidence". Deciding whether /dev/kfd belongs in the CLOSED list needs them
+ apart: a topology that reads and reports only NVIDIA (vendor 4318) or the CPU node is
+ positive evidence the node is not AMD's to repair, while one that cannot be read is no
+ evidence either way and is exactly what a container hiding the sysfs while mapping the
+ node produces.
+ """
+ nodes = "/sys/class/kfd/kfd/topology/nodes"
+ try:
+ entries = os.listdir(nodes)
+ except OSError:
+ return None
+ _read_one = False
+ _missed_one = False
+ for entry in entries:
+ try:
+ with open(os.path.join(nodes, entry, "properties"), encoding = "utf-8") as fh:
+ properties = fh.read()
+ except (OSError, UnicodeDecodeError):
+ _missed_one = True
+ continue
+ _read_one = True
+ if re.search(r"\bvendor_id\s+4098\b", properties):
+ return True
+ # A directory that lists but whose properties will not open says nothing either, and
+ # neither does a PARTIAL read: the CPU node opening while the GPU node does not is one
+ # entry short of "names none", and False there drops /dev/kfd from the closed list, so a
+ # host whose KFD is video and whose render node is render is told to join only render
+ # and left with KFD shut. install.sh's _kfd_gfx_targets states the same rule.
+ return None if _missed_one or not _read_one else False
+
+
+def _a_confirmed_amd_render_node_exists() -> bool:
+ """Whether DRM names an AMD render node outright, with its vendor actually read.
+
+ The strict counterpart to _amd_render_node_exists, which reads an unreadable vendor as
+ present on purpose. Nothing here may be assumed: this is used as INDEPENDENT evidence
+ of AMD silicon where KFD's topology cannot be read, so an unknown vendor would let an
+ NVIDIA-only host claim an AMD node.
+ """
+ return any(_render_node_is_amd(path) for path in glob.glob(_DRI_RENDER_GLOB))
+
+
+def _kfd_topology_has_an_amd_gpu() -> bool:
+ """Whether KFD enumerates an AMD GPU node, so ``/dev/kfd`` is one worth opening.
+
+ Mirrors ``hardware._linux_kfd_reports_an_amd_gpu``: gpu_id 0 is the CPU node, and
+ NVIDIA's open kernel module registers KFD nodes of its own under vendor_id 4318,
+ so AMD ownership is confirmed rather than assumed.
+ """
+ return _kfd_topology_amd_state() is True
+
+
+def amd_kfd_gpu_node_count() -> Optional[int]:
+ """How many AMD GPU agents KFD enumerates, or ``None`` when that cannot be read.
+
+ The ordinal space a visibility mask indexes: HIP numbers GPU agents, so the CPU node
+ every KFD topology carries is excluded twice over -- it reports ``vendor_id 0``, the
+ guard install.sh already relies on, and a ``simd_count`` of zero. A node that reports
+ no ``simd_count`` at all is counted, since dropping it would understate the bound and
+ an understated bound is what calls a valid selector a blocker. Read from
+ world-readable sysfs, so it answers on the closed-node host this module exists for.
+
+ ``None`` and 0 are both "unknown" to callers by design -- a topology that is missing,
+ unreadable, or reports no GPU bounds nothing, and treating it as a bound would make
+ every selector on the host look like it names a device that is not there.
+ """
+ nodes = "/sys/class/kfd/kfd/topology/nodes"
+ try:
+ entries = os.listdir(nodes)
+ except OSError:
+ return None
+ count = 0
+ for entry in entries:
+ try:
+ with open(os.path.join(nodes, entry, "properties"), encoding = "utf-8") as fh:
+ properties = fh.read()
+ except (OSError, UnicodeDecodeError):
+ # Unknown propagates, exactly as _amd_render_node_exists treats an unreadable
+ # vendor. Skipping the entry would answer with a SMALLER count on a multi-GPU
+ # host where one topology entry is momentarily unreadable, and an understated
+ # bound is the thing this function's own contract says calls a valid selector
+ # a blocker: HIP_VISIBLE_DEVICES=1 against a count of 1 reads as hiding every
+ # device, and the user is sent to clear a mask that hides nothing.
+ return None
+ if not re.search(r"\bvendor_id\s+4098\b", properties):
+ continue
+ _simd = re.search(r"\bsimd_count\s+(\d+)\b", properties)
+ if _simd is None or int(_simd.group(1)) > 0:
+ count += 1
+ return count
+
+
+def _amd_render_node_exists() -> bool:
+ """Whether any AMD render node is present at all.
+
+ Presence, not openability. A container given ``--device /dev/kfd`` and not
+ ``--device /dev/dri`` passes every probe in this file, and ROCr opens a render node
+ to talk to amdgpu, so it initialises nothing; ``docker/run.sh`` passes both devices
+ for that reason. Group membership cannot create the node, so this is an independent
+ blocker rather than part of the permission repair.
+ """
+ _unreadable = False
+ for path in glob.glob(_DRI_RENDER_GLOB):
+ _vendor = _render_node_vendor(path)
+ if _vendor == _AMD_PCI_VENDOR_ID:
+ return True
+ _unreadable = _unreadable or _vendor is None
+ # A node whose vendor could not be READ is not evidence that no AMD node exists, and the
+ # caller turns "does not exist" into "recreate the container with --device /dev/dri" --
+ # advice for a device that shape has already mapped. Unknown reads as present, which
+ # withdraws a sentence rather than inventing one.
+ return _unreadable
+
+
+def an_amd_render_node_is_open() -> bool:
+ """Whether this user can open at least one AMD render node.
+
+ The counterpart to amd_nodes_closed_to_this_user, and the reason it is not simply
+ "closed is empty": a multi-AMD host can have one node shut and another open, and a
+ caller explaining an empty GPU probe needs to know that the runtime had a node to
+ use. False off Linux, where there are no such nodes to open.
+ """
+ if platform.system() != "Linux":
+ return False
+ for path in sorted(glob.glob(_DRI_RENDER_GLOB)):
+ try:
+ if not _render_node_is_amd(path):
+ continue
+ if os.access(path, os.R_OK | os.W_OK):
+ return True
+ except OSError:
+ continue
+ return False
+
+
+# RADV radeon_icd.x86_64.json, AMDVLK amd_icd64 / amd_pro_icd64 / amdvlk64, Adrenalin
+# amd-vulkan64.json. install_llama_prebuilt._AMD_VULKAN_ICD_NEEDLES is the same list for
+# the same reason and a test below holds the two together; it is copied rather than
+# imported so the inference path does not pull in the installer.
+_AMD_VULKAN_ICD_NEEDLES = ("radeon", "radv", "amdvlk", "amd_icd", "amd_pro", "amd_vulkan")
+
+# Mesa and AMDVLK both register a 32-bit manifest beside the 64-bit one, and a 64-bit
+# llama-server cannot load either vendor's. Same rule and same needles as
+# install_llama_prebuilt._is_amd_64_bit, which rejects them for the same reason; a test
+# holds the two lists together.
+_VULKAN_ICD_32_BIT_NEEDLES = ("i686", "i386")
+
+
+def _vulkan_glob_matches(pattern: str, name: str) -> bool:
+ """The loader's four driver-filter globs, case-insensitively: "s", "s*", "*s", "*s*"."""
+ pattern, name = pattern.lower(), name.lower()
+ starts, ends = pattern.startswith("*"), pattern.endswith("*")
+ core = pattern[1 if starts else 0 : len(pattern) - 1 if ends else len(pattern)]
+ if starts and ends:
+ return core in name
+ if starts:
+ return name.endswith(core)
+ if ends:
+ return name.startswith(core)
+ return name == core
+
+
+def _vulkan_loader_allows(path: str) -> bool:
+ """Whether the loader's own driver filters leave this manifest loadable.
+
+ They apply to every driver the loader knows, a forced list included, and match the
+ manifest's basename. Select is an allowlist and disable is a denylist that WINS over it:
+ "The values from the disable environment variable will be considered before the enable
+ or select environment variable", and VK_LOADER_DRIVERS_DISABLE is "also checked before
+ other driver environment variables (such as VK_LOADER_DRIVERS_SELECT)" (Vulkan-Loader,
+ LoaderInterfaceArchitecture.md). Drivers have no VK_LOADER_LAYERS_ALLOW counterpart to
+ name one back, so selecting radeon* while also disabling radeon* leaves the loader with
+ no driver at all, where treating a set select list as the whole answer counted Radeon as
+ usable and reported only the device-node repair.
+
+ install_llama_prebuilt._vulkan_loader_allows is the same rule for the same reason; a
+ test below runs the two against one table so they cannot drift.
+ """
+
+ def _globs(env_name: str) -> "list[str]":
+ value = os.environ.get(env_name) or ""
+ return [entry.strip() for entry in value.split(",") if entry.strip()]
+
+ name = PurePath(path).name
+ disable = _globs("VK_LOADER_DRIVERS_DISABLE")
+ if any(_vulkan_glob_matches(pattern, name) for pattern in disable):
+ return False
+ select = _globs("VK_LOADER_DRIVERS_SELECT")
+ return any(_vulkan_glob_matches(pattern, name) for pattern in select) if select else True
+
+
+# ld.so's own defaults, plus the multiarch directories Debian and Ubuntu install into.
+_DEFAULT_LIBRARY_DIRS = (
+ "/lib",
+ "/lib64",
+ "/usr/lib",
+ "/usr/lib64",
+ "/usr/local/lib",
+ "/usr/local/lib64",
+)
+
+_LD_SO_CONF = "/etc/ld.so.conf"
+
+_ld_cache_sonames_cached: "frozenset[str] | None" = None
+_ld_cache_read = False
+
+
+def _ld_so_conf_dirs(path: str = _LD_SO_CONF, _seen: "set[str] | None" = None) -> "list[str]":
+ """The extra library directories /etc/ld.so.conf names, include lines followed.
+
+ Read because a driver installed outside the defaults is the ordinary shape for a
+ vendor package -- amdgpu-pro puts its libraries under /opt -- and missing that
+ directory would make a live driver look like a stale registration.
+ """
+ _seen = set() if _seen is None else _seen
+ if path in _seen:
+ return []
+ _seen.add(path)
+ dirs: "list[str]" = []
+ try:
+ with open(path, "r", encoding = "utf-8", errors = "replace") as handle:
+ lines = handle.read().splitlines()
+ except OSError:
+ return dirs
+ for line in lines:
+ line = line.split("#", 1)[0].strip()
+ if not line:
+ continue
+ if line.startswith("include"):
+ for entry in sorted(glob.glob(line[len("include") :].strip())):
+ dirs.extend(_ld_so_conf_dirs(entry, _seen))
+ continue
+ dirs.append(line)
+ return dirs
+
+
+def _dynamic_loader_search_dirs() -> "list[str]":
+ """Where ld.so would look for a bare soname here, in its own order."""
+ dirs = [
+ entry
+ for entry in (os.environ.get("LD_LIBRARY_PATH") or "").split(os.pathsep)
+ if entry.strip()
+ ]
+ dirs.extend(_ld_so_conf_dirs())
+ dirs.extend(_DEFAULT_LIBRARY_DIRS)
+ for pattern in ("/usr/lib/*-linux-gnu*", "/lib/*-linux-gnu*"):
+ dirs.extend(sorted(glob.glob(pattern)))
+ return list(dict.fromkeys(dirs))
+
+
+def _ld_cache_sonames() -> "frozenset[str] | None":
+ """Every soname in the loader's cache, or None when the cache cannot be read.
+
+ None is not an empty set: musl ships no `ldconfig -p` and a minimal container may ship
+ no ldconfig at all, and answering "nothing is installed" there would call every bare
+ registration stale. Read once, since the diagnosis asks it per manifest.
+ """
+ global _ld_cache_sonames_cached, _ld_cache_read
+
+ if _ld_cache_read:
+ return _ld_cache_sonames_cached
+ _ld_cache_read = True
+ _ld_cache_sonames_cached = None
+ for _candidate in ("ldconfig", "/sbin/ldconfig", "/usr/sbin/ldconfig"):
+ _exe = shutil.which(_candidate) if "/" not in _candidate else _candidate
+ if not _exe or not os.path.exists(_exe):
+ continue
+ try:
+ _out = subprocess.run(
+ [_exe, "-p"],
+ capture_output = True,
+ text = True,
+ # Named rather than inherited, as every other call here does: the default
+ # is locale.getencoding(), which is ASCII under the C locale a CI runner
+ # or container routinely has, and a library path outside it would then
+ # raise instead of being read.
+ encoding = "utf-8",
+ errors = "replace",
+ timeout = 10,
+ **windows_hidden_subprocess_kwargs(),
+ )
+ except (OSError, subprocess.SubprocessError):
+ continue
+ if _out.returncode != 0:
+ continue
+ # "\tlibfoo.so.1 (libc6,x86-64) => /usr/lib/libfoo.so.1"
+ _names = {
+ line.strip().split(" ", 1)[0]
+ for line in (_out.stdout or "").splitlines()
+ if "=>" in line and line.strip()
+ }
+ # Unconditionally, including the empty case: glibc's ldconfig prints
+ # "0 libs found in cache" and exits 0 for a cache that is present and empty, and
+ # exits 1 with nothing on stdout when the cache file is absent. The non-zero arm
+ # above is what separates those, so storing only a non-empty set collapsed the
+ # distinction this function's contract is built on -- a fresh container whose cache
+ # has not been built read as "cannot enumerate", and _a_bare_soname_resolves then
+ # answered True for a soname that is on no search path and in no cache, which
+ # withholds the reinstall half of the repair rather than offering it.
+ _ld_cache_sonames_cached = frozenset(_names)
+ return _ld_cache_sonames_cached
+ return _ld_cache_sonames_cached
+
+
+def _a_bare_soname_resolves(soname: str) -> bool:
+ """Whether a manifest's bare library name still resolves to something on this host.
+
+ A manifest may name its library by soname alone and leave the loader to find it, which
+ is what NVIDIA's registration does, so a bare name cannot simply be trusted: the
+ package can be removed and leave the manifest behind, and the loader then has one fewer
+ driver than the registration count suggests.
+
+ Positive evidence in the negative direction as well, since both answers are load
+ bearing. Found on disk or in the loader's cache is a driver. NOT found decides the
+ question only when the cache could actually be read; a host whose loader configuration
+ this cannot enumerate answers True, because calling a live driver stale would demote
+ the node hint on a host whose other vendor really does have a path.
+ """
+ for _directory in _dynamic_loader_search_dirs():
+ try:
+ if os.path.isfile(os.path.join(_directory, soname)):
+ return True
+ except OSError:
+ continue
+ _cache = _ld_cache_sonames()
+ if _cache is None:
+ return True
+ return soname in _cache
+
+
+def _icd_manifest_is_usable(path: str) -> bool:
+ """Whether a manifest still points at a driver library that is there.
+
+ A leftover or malformed JSON is a registration with no device behind it. A bare soname
+ is resolved rather than assumed: it is the form NVIDIA registers under, and a removed
+ package leaves the manifest behind, so trusting the name counted a driver that is not
+ there and withheld the reinstall half of the repair.
+
+ The three fields checked are the three loader_parse_icd_manifest skips the file for:
+ a missing or non-string file_format_version, a missing ICD.library_path, and a missing
+ or non-string ICD.api_version. An UNRECOGNISED file_format_version is deliberately not
+ one of them -- the loader only logs "may cause errors" there and carries on, so
+ refusing it would drop a driver the loader loads.
+ """
+ try:
+ with open(path, "r", encoding = "utf-8") as handle:
+ manifest = json.load(handle)
+ icd = manifest.get("ICD") or {}
+ version = manifest.get("file_format_version")
+ library = icd.get("library_path")
+ api = icd.get("api_version")
+ except Exception: # noqa: BLE001
+ return False
+ if not isinstance(version, str) or not version.strip():
+ return False
+ if not isinstance(api, str) or not api.strip():
+ return False
+ if not isinstance(library, str) or not library.strip():
+ return False
+ library = library.strip()
+ if not (os.path.isabs(library) or "/" in library or "\\" in library):
+ return _a_bare_soname_resolves(library)
+ if not os.path.isabs(library):
+ # Relative to the manifest's directory, per the loader's interface document.
+ library = os.path.join(os.path.dirname(path), library)
+ try:
+ return os.path.isfile(library)
+ except OSError:
+ return False
+
+
+def _is_an_amd_icd_name(path: str) -> bool:
+ """Whether a manifest's own filename is one an AMD driver registers under."""
+ stem = PurePath(path).stem.lower().replace("-", "_")
+ return any(needle in stem for needle in _AMD_VULKAN_ICD_NEEDLES)
+
+
+def _is_a_32_bit_icd_name(path: str) -> bool:
+ """Whether a manifest's own filename marks it as the 32-bit build of a driver.
+
+ Asked of EVERY vendor rather than of AMD alone, unlike the installer's copy: the
+ question there is "is an AMD driver installed", and here it is "what can this binary
+ load", which a 32-bit NVIDIA or Intel manifest answers no to just as squarely.
+ """
+ stem = PurePath(path).stem.lower().replace("-", "_")
+ return stem.endswith("32") or any(n in stem for n in _VULKAN_ICD_32_BIT_NEEDLES)
+
+
+def _icd_library_path(path: str) -> "str | None":
+ """The library file a manifest points at, when it can be found on disk.
+
+ Separate from _icd_manifest_is_usable, which asks whether the loader has SOMETHING to
+ load: this asks which file, so the file itself can be read.
+ """
+ try:
+ with open(path, "r", encoding = "utf-8") as handle:
+ library = (json.load(handle).get("ICD") or {}).get("library_path")
+ except Exception: # noqa: BLE001
+ return None
+ if not isinstance(library, str) or not library.strip():
+ return None
+ library = library.strip()
+ if os.path.isabs(library) or "/" in library or "\\" in library:
+ if not os.path.isabs(library):
+ library = os.path.join(os.path.dirname(path), library)
+ try:
+ return library if os.path.isfile(library) else None
+ except OSError:
+ return None
+ # Every match, not the first: a multilib host carries the same soname in both
+ # bitnesses, ld.so picks the one matching the process, and the search order does not.
+ # sorted(glob) puts /usr/lib/i386-linux-gnu ahead of x86_64, so taking the first hit
+ # handed the 32-bit copy to _an_icd_is_32_bit and discarded a driver the loader loads.
+ _first: "str | None" = None
+ for _directory in _dynamic_loader_search_dirs():
+ _candidate = os.path.join(_directory, library)
+ try:
+ if not os.path.isfile(_candidate):
+ continue
+ except OSError:
+ continue
+ if _library_file_is_32_bit(_candidate) is False:
+ return _candidate
+ if _first is None:
+ _first = _candidate
+ # No 64-bit copy: hand back whatever is there, so a genuinely 32-bit-only
+ # registration is still read from the object rather than from its filename.
+ return _first
+
+
+def _icd_manifest_declares_32_bit(path: str) -> "bool | None":
+ """The manifest's own architecture claim, or None where it makes none.
+
+ ICD.library_arch is the loader's own field, a string "32" or "64", and the loader reads
+ it for exactly this purpose: to skip a driver whose bitness cannot match the process
+ before trying to open it. Optional, and Debian strips it back out of Mesa's manifests
+ to keep one file across architectures, so its absence is ordinary and decides nothing.
+ """
+ try:
+ with open(path, "r", encoding = "utf-8") as handle:
+ declared = (json.load(handle).get("ICD") or {}).get("library_arch")
+ except Exception: # noqa: BLE001
+ return None
+ if not isinstance(declared, str):
+ return None
+ declared = declared.strip()
+ if declared == "32":
+ return True
+ if declared == "64":
+ return False
+ return None
+
+
+def _library_file_is_32_bit(path: str) -> "bool | None":
+ """The ELF class of a library file, or None where the file does not say.
+
+ e_ident[EI_CLASS] is byte 4 of every ELF file and is 1 for 32-bit, 2 for 64-bit. Read
+ rather than inferred, so a manifest that declares nothing and is named neutrally is
+ still answered by the object itself.
+ """
+ try:
+ with open(path, "rb") as handle:
+ header = handle.read(5)
+ except OSError:
+ return None
+ if len(header) < 5 or header[:4] != b"\x7fELF":
+ return None
+ if header[4] == 1:
+ return True
+ if header[4] == 2:
+ return False
+ return None
+
+
+def _an_icd_is_32_bit(path: str) -> bool:
+ """Whether this manifest registers a driver a 64-bit process cannot load.
+
+ Three sources in order of how much they know. The manifest's declared library_arch is
+ the loader's own answer. Failing that the library's ELF class is the object's own, which
+ covers the case a filename cannot: Mesa's manifests carry no marker once Debian has
+ rewritten them. The filename needles are the last resort, and the only one the installer
+ has, since it decides this before any library is resolvable.
+
+ A 64-bit process is assumed, which is what Studio ships; on a 32-bit build the question
+ inverts, and no build of that shape exists here.
+ """
+ declared = _icd_manifest_declares_32_bit(path)
+ if declared is not None:
+ return declared
+ library = _icd_library_path(path)
+ if library is not None:
+ _elf = _library_file_is_32_bit(library)
+ if _elf is not None:
+ return _elf
+ return _is_a_32_bit_icd_name(path)
+
+
+def _vulkan_icd_search_dirs() -> "list[str]":
+ """The icd.d directories the loader would search, in its own order.
+
+ From the XDG variables, since the loader falls back to the defaults only when one is
+ unset: reading the defaults regardless both misses a custom layout's only manifest and
+ counts stale ones the loader would never read.
+ install_llama_prebuilt._vulkan_icd_search_dirs is the same list, and a test holds them
+ together.
+ """
+
+ def _paths(var: str, default: str) -> "list[str]":
+ value = os.environ.get(var)
+ raw = value if (value or "").strip() else default
+ return [entry for entry in raw.split(os.pathsep) if entry.strip()]
+
+ def _home(var: str, default: str) -> "list[str]":
+ value = os.environ.get(var)
+ if (value or "").strip():
+ return [value]
+ try:
+ return [os.path.expanduser(os.path.join("~", default))]
+ except Exception: # noqa: BLE001
+ return []
+
+ dirs = [
+ *(
+ os.path.join(base, "vulkan/icd.d")
+ for base in (
+ *_home("XDG_CONFIG_HOME", ".config"),
+ *_paths("XDG_CONFIG_DIRS", "/etc/xdg"),
+ )
+ ),
+ "/etc/vulkan/icd.d",
+ *(
+ os.path.join(base, "vulkan/icd.d")
+ for base in (
+ *_home("XDG_DATA_HOME", ".local/share"),
+ *_paths("XDG_DATA_DIRS", "/usr/local/share" + os.pathsep + "/usr/share"),
+ )
+ ),
+ ]
+ return list(dict.fromkeys(dirs))
+
+
+def _vulkan_icd_manifest_paths() -> "list[str]":
+ """Every ICD manifest the loader knows about here, before its filters are applied.
+
+ A forced list REPLACES the search rather than adding to it, and VK_DRIVER_FILES
+ supersedes VK_ICD_FILENAMES rather than joining it. VK_ADD_DRIVER_FILES is the additive
+ one: the loader reads it FIRST and then the search, and ignores it entirely when either
+ force list is set. It may name a manifest no search directory holds, so leaving it out
+ made a host with an added driver look like one that has only what the walk found.
+
+ Linux only, since the render nodes this is asked about exist nowhere else.
+ """
+ for var in ("VK_DRIVER_FILES", "VK_ICD_FILENAMES"):
+ if (os.environ.get(var) or "").strip():
+ return _forced_icd_manifest_paths(var)
+ return _searched_vulkan_icd_manifest_paths()
+
+
+def _searched_vulkan_icd_manifest_paths() -> "list[str]":
+ """The half of the above a forced list REPLACES: the search, plus the additive list.
+
+ Split out so the attribution below can ask what clearing a forced list would expose.
+ """
+ if platform.system() != "Linux":
+ return []
+ added = (os.environ.get("VK_ADD_DRIVER_FILES") or "").strip()
+ paths = [entry.strip() for entry in added.split(os.pathsep) if entry.strip()]
+ for directory in _vulkan_icd_search_dirs():
+ try:
+ paths.extend(sorted(glob.glob(os.path.join(directory, "*.json"))))
+ except OSError:
+ continue
+ return list(dict.fromkeys(paths))
+
+
+def the_vulkan_loader_can_only_load_amd() -> bool:
+ """Whether every driver this loader would actually load is an AMD one.
+
+ Then no other vendor's driver is ever opened, so its render node is not a path this
+ binary has however open it is. Three things decide it and all three are the loader's
+ own: which manifests it looks at (a forced list, else the search dirs), its driver
+ filters, and whether each manifest still resolves to a library.
+
+ POSITIVE evidence only, so both "nothing could be enumerated" and "the filters leave no
+ driver at all" answer False. The second is not an oversight: a loader with no driver
+ explains an empty probe by itself, and the closed AMD node is then not the cause either.
+ """
+ loadable = _loadable_icd_manifests()
+ if not loadable:
+ return False
+ return all(_is_an_amd_icd_name(path) for path in loadable)
+
+
+def _loadable_icd_manifests(paths: "list[str] | None" = None) -> "list[str]":
+ """The manifests the loader would both find here and load; ``paths`` asks of a
+ candidate list instead of this host's."""
+ return [
+ path
+ for path in (_vulkan_icd_manifest_paths() if paths is None else paths)
+ # A 32-bit manifest is registered beside the 64-bit one and this binary cannot load
+ # it, so it is neither evidence of an AMD driver nor of another vendor's.
+ if not _an_icd_is_32_bit(path)
+ and _vulkan_loader_allows(path)
+ and _icd_manifest_is_usable(path)
+ ]
+
+
+def the_vulkan_loader_has_no_usable_driver() -> bool:
+ """Whether the loader would find driver manifests here and load none of them.
+
+ A second blocker rather than a competing explanation: a removed library, a filter that
+ disables the last driver, or a 32-bit-only registration leaves the probe empty however
+ the render node is owned, so opening the node repairs nothing on its own.
+
+ Positive evidence only, and the two failing answers are different. An enumeration that
+ found NOTHING says only that this cannot read the loader's configuration -- a registry
+ layout, a distribution that registers drivers some other way -- so it answers False. An
+ enumeration that found manifests and could load none of them is the claim itself.
+ """
+ paths = _vulkan_icd_manifest_paths()
+ if not paths:
+ return False
+ return not _loadable_icd_manifests()
+
+
+_DRIVER_OVERRIDES = (
+ "VK_DRIVER_FILES",
+ "VK_ICD_FILENAMES",
+ "VK_LOADER_DRIVERS_SELECT",
+ "VK_LOADER_DRIVERS_DISABLE",
+)
+
+
+def _vulkan_override_patterns(var: str) -> "list[str]":
+ value = os.environ.get(var) or ""
+ return [entry.strip() for entry in value.split(",") if entry.strip()]
+
+
+def _the_loader_would_have_a_driver_without(cleared: "frozenset[str]") -> bool:
+ """Whether the loader would end up with a usable driver if ``cleared`` were unset.
+
+ Asked once rather than restated per variable: a repair only counts if it leaves a
+ driver the loader can LOAD. Filename allowance alone named a filter over a manifest
+ whose library was gone, where clearing it changes nothing and the fix is a reinstall.
+ """
+ forced = [var for var in ("VK_DRIVER_FILES", "VK_ICD_FILENAMES") if _is_set(var)]
+ if not set(forced) & cleared:
+ candidates = _vulkan_icd_manifest_paths()
+ else:
+ remaining = [var for var in forced if var not in cleared]
+ candidates = (
+ _forced_icd_manifest_paths(remaining[0])
+ if remaining
+ else _searched_vulkan_icd_manifest_paths()
+ )
+ disable = (
+ []
+ if "VK_LOADER_DRIVERS_DISABLE" in cleared
+ else _vulkan_override_patterns("VK_LOADER_DRIVERS_DISABLE")
+ )
+ select = (
+ []
+ if "VK_LOADER_DRIVERS_SELECT" in cleared
+ else _vulkan_override_patterns("VK_LOADER_DRIVERS_SELECT")
+ )
+ for path in candidates:
+ name = PurePath(path).name
+ if any(_vulkan_glob_matches(pattern, name) for pattern in disable):
+ continue
+ if select and not any(_vulkan_glob_matches(pattern, name) for pattern in select):
+ continue
+ if not _an_icd_is_32_bit(path) and _icd_manifest_is_usable(path):
+ return True
+ return False
+
+
+def _is_set(var: str) -> bool:
+ return bool((os.environ.get(var) or "").strip())
+
+
+def _forced_icd_manifest_paths(var: str) -> "list[str]":
+ value = (os.environ.get(var) or "").strip()
+ return [entry.strip() for entry in value.split(os.pathsep) if entry.strip()]
+
+
+def the_vulkan_loader_override_to_blame() -> "str | None":
+ """The environment variable to name when the loader can load none of its manifests.
+
+ ``the_vulkan_loader_has_no_usable_driver`` answers for three causes and only one is
+ repaired by installing a driver. A filter that disables every manifest, and a forced
+ list pointing at paths that do not resolve, are settings: reinstalling leaves the
+ variable in place and the probe just as empty.
+
+ Decided by the counterfactual rather than by any precedence between the variables:
+ the smallest set whose removal leaves the loader a driver it can LOAD. One variable
+ if one is enough, all of them together when no single removal helps, and ``None``
+ when removing every override still leaves nothing -- which is the missing-library and
+ 32-bit case the reinstall sentence was written for.
+
+ Order within a combined answer follows ``_DRIVER_OVERRIDES``, not the environment.
+ """
+ if not _vulkan_icd_manifest_paths():
+ return None
+ overrides = [var for var in _DRIVER_OVERRIDES if _is_set(var)]
+ if not overrides:
+ return None
+ for var in overrides:
+ if _the_loader_would_have_a_driver_without(frozenset([var])):
+ return var
+ if len(overrides) > 1 and _the_loader_would_have_a_driver_without(frozenset(overrides)):
+ return " and ".join(overrides) + " together"
+ return None
+
+
+def a_non_amd_render_node_is_open() -> bool:
+ """Whether a render node belonging to some OTHER vendor is open to this user.
+
+ Vulkan enumerates any vendor, so on a mixed host an open Intel or NVIDIA render node is
+ a complete path for a Vulkan-only build: a closed AMD node is then a second finding
+ rather than why the probe came back empty. HIP has no such alternative, which is why
+ the caller asks this only for Vulkan. Only nodes whose vendor was READ count.
+ """
+ if platform.system() != "Linux":
+ return False
+ for path in sorted(glob.glob(_DRI_RENDER_GLOB)):
+ _vendor = _render_node_vendor(path)
+ if _vendor is None or _vendor == _AMD_PCI_VENDOR_ID:
+ continue
+ try:
+ if os.access(path, os.R_OK | os.W_OK):
+ return True
+ except OSError:
+ continue
+ return False
+
+
+def amd_nodes_closed_to_this_user() -> list[str]:
+ """AMD device nodes that exist on this host and this user cannot open.
+
+ Every AMD probe in this tree tests that the node EXISTS. On a stock Linux distribution
+ these are ``root:render`` mode 0660, so a user outside that group passes all of them
+ and then cannot open the device: HIP counts zero devices, the Vulkan loader enumerates
+ none, and both look exactly like owning no GPU. That is the whole of #10466.
+
+ Only AMD-owned nodes count, since every vendor's render node has these permissions and
+ an NVIDIA box would report an identical closed list with no such problem.
+
+ ``os.access`` rather than a trial ``open()``: opening ``/dev/kfd`` initialises KFD state
+ for the process, which the probes calling this exist to avoid. Empty off Linux, on a
+ host with no AMD nodes, and for root.
+ """
+ if platform.system() != "Linux":
+ return []
+ closed = []
+ _amd_in_topology = None
+ for path in [_KFD_NODE, *sorted(glob.glob(_DRI_RENDER_GLOB))]:
+ try:
+ if not os.path.exists(path) or os.access(path, os.R_OK | os.W_OK):
+ continue
+ except OSError:
+ continue
+ if _amd_in_topology is None:
+ _amd_in_topology = _kfd_topology_has_an_amd_gpu()
+ if path == _KFD_NODE:
+ # DRM is independent evidence of the same silicon, and it is only consulted
+ # where KFD's own topology could not be read at all -- a readable topology
+ # naming no AMD GPU still excludes the node, which is what keeps an NVIDIA-only
+ # host silent. Without the fallback a host whose sysfs is hidden reported only
+ # the render node, and where the two carry different owning groups (video
+ # against render) the hint named a membership that leaves KFD shut and ROCm
+ # with nothing to open. Vendor-confirmed, never assumed: an unreadable render
+ # vendor is not evidence of AMD either.
+ if _amd_in_topology or (
+ _kfd_topology_amd_state() is None and _a_confirmed_amd_render_node_exists()
+ ):
+ closed.append(path)
+ continue
+ _vendor = _render_node_vendor(path)
+ if _vendor == _AMD_PCI_VENDOR_ID:
+ closed.append(path)
+ elif _vendor is None and _amd_in_topology:
+ # A container can map the node and hide the sysfs entry that names its vendor.
+ # Dropping it there left a host with a shut node reporting nothing closed, while
+ # _amd_render_node_exists reads the same unknown as PRESENT and withdraws the
+ # missing-node sentence too -- so #10466's own shape got no diagnosis at all.
+ # KFD is the independent evidence, and it is what keeps an NVIDIA-only host
+ # silent: its topology reports vendor 0x10DE, so this arm is never reached.
+ closed.append(path)
+ return closed
+
+
+def _has_an_access_acl(path: str) -> bool:
+ """Whether ``path`` carries a POSIX access ACL, so its mode bits are the ACL mask.
+
+ Read through the xattr rather than by shelling out to ``getfacl``, which is not
+ installed everywhere this runs. False on any platform or filesystem that cannot
+ answer, which is the direction that keeps the ordinary node prescribed for.
+
+ os.listxattr returns the names as ``str`` for a ``str`` path, so a bytes literal can
+ never match one and the check would be dead. Both are accepted rather than assumed,
+ because a bytes path yields bytes names and the caller decides the path type.
+ """
+ try:
+ names = os.listxattr(path)
+ except (OSError, AttributeError, UnicodeDecodeError):
+ return False
+ return any(
+ (_n.decode("utf-8", "replace") if isinstance(_n, bytes) else _n)
+ == "system.posix_acl_access"
+ for _n in names
+ )
+
+
+# Groups whose membership reaches far beyond a device node. Not exhaustive and does not
+# need to be: anything here is reported instead of prescribed, and an unlisted group that
+# turns out to be privileged is the status quo rather than a regression.
+# docker and lxd are here for the same reason as wheel: membership is root by another
+# route, since either can start a container or VM with the host filesystem mounted. A node
+# owned by one is a udev mistake to report rather than a group to join for a GPU.
+_PRIVILEGED_GROUPS = frozenset(
+ {
+ "root",
+ "wheel",
+ "sudo",
+ "admin",
+ "adm",
+ "disk",
+ "kmem",
+ "shadow",
+ "docker",
+ "lxd",
+ }
+)
+
+
+# A name safe to paste into usermod, narrower than one NSS can return: groupadd(8)'s
+# portable set plus the trailing $ of a Samba machine account. Anything else is reported by
+# GID, because two layers below read such a name as STRUCTURE: usermod -G splits on commas
+# inside itself, after the shell has handed it one argument, so "render,sudo" is two groups
+# and _PRIVILEGED_GROUPS never matches; and install.sh's twin parses stat output with
+# awk -F'|'. --group-add takes the GID anyway, so nothing is lost.
+_GROUP_NAME_RE = re.compile(r"[A-Za-z_][A-Za-z0-9_.-]*\$?\Z")
+
+
+def _group_name_is_prescribable(name: str) -> bool:
+ """Whether ``name`` can be pasted into a repair command as a single group."""
+ return bool(name) and _GROUP_NAME_RE.match(name) is not None
+
+
+def _groups_that_own(paths: list) -> tuple:
+ """How to open ``paths``, read from the nodes themselves.
+
+ "render,video" is not always the right pair, and sometimes no group is the answer at
+ all, so the seven buckets returned each carry a different repair:
+
+ ``joinable`` membership WOULD open it, so ``usermod -a -G`` is the fix.
+ ``unnamed`` GIDs with no entry in the group database, the container case
+ ``docker/run.sh`` documents. usermod refuses a bare GID (shadow 4.13:
+ ``group '993' does not exist``, exit 6), so these are reported.
+ ``no_group`` the mode denies the group too, e.g. a udev rule leaving one
+ ``root:render 0600``. Joining render there changes nothing.
+ ``acl`` a POSIX access ACL, where the mode's group bits are the ACL mask and
+ the real grant is undecidable from a stat.
+ ``owned`` this account owns it, and POSIX stops at the owner class once the uid
+ matches, so no membership opens it however the group bits read.
+ ``privileged`` the owning group grants far more than the GPU: a udev misconfiguration
+ to report rather than a membership to prescribe.
+ ``already`` already in the group and still shut, so a container device cgroup or an
+ LSM denies it and usermod would exit 0 and change nothing.
+ ``external`` this account owns it AND the owner bits already grant read and write,
+ so the mode is not what is shutting it either: same external denial as
+ ``already``, reached by owner class rather than by membership.
+
+ Best effort: a node that cannot be stat'd joins no bucket rather than raising.
+ """
+ joinable, unnamed, no_group, acl, owned, privileged, already, external = (
+ [],
+ [],
+ [],
+ [],
+ [],
+ [],
+ [],
+ [],
+ )
+ try:
+ # The account's own gids, read once. getgroups() is the supplementary list and does
+ # not always include the primary one, so both are needed.
+ _mine = {os.getgid(), *os.getgroups()}
+ except (OSError, AttributeError):
+ _mine = set()
+ for path in paths:
+ try:
+ _st = os.stat(path)
+ except OSError:
+ continue
+ # acl(5): once a node carries an access ACL, the group-class bits in st_mode are
+ # the ACL MASK rather than the owning group's grant, so every reading below is of
+ # an upper bound. The mask can allow rw while the group entry denies it, and a
+ # named-group entry can grant what the mode hides. Neither is decidable without
+ # parsing the ACL, so such a node is reported rather than prescribed for.
+ if _has_an_access_acl(path):
+ acl.append(path)
+ continue
+ # POSIX resolves the owner class EXCLUSIVELY once the uid matches, so on a node
+ # this account owns the group bits are never consulted and joining the group
+ # cannot open it. os.access() already said it is shut; the repair is the mode.
+ if _st.st_uid == os.getuid():
+ # Unless the owner bits already grant rw: the node is known shut, so the
+ # mode is not what denies it. Same external denial as `already`, by owner class.
+ if _st.st_mode & stat.S_IRUSR and _st.st_mode & stat.S_IWUSR:
+ external.append(path)
+ else:
+ owned.append(path)
+ continue
+ # Neither owner nor group member puts this account in the OTHER class, which POSIX
+ # also resolves exclusively: bits already granting rw mean the mode is not what
+ # denies a node os.access() just called shut. Same rule as the owner and
+ # already-a-member branches, for the third class. Guarded on membership (a member is
+ # in the GROUP class, where `already` names the group more precisely) and above the
+ # group-bits test, which would otherwise prescribe a chmod that cannot help.
+ if _st.st_gid not in _mine and (_st.st_mode & stat.S_IROTH and _st.st_mode & stat.S_IWOTH):
+ external.append(path)
+ continue
+ # Group read AND write: HIP and the Vulkan loader both open the node read-write,
+ # which is the same bar amd_nodes_closed_to_this_user() applied to this account.
+ if (_st.st_mode & stat.S_IRGRP) == 0 or (_st.st_mode & stat.S_IWGRP) == 0:
+ no_group.append(path)
+ continue
+ try:
+ import grp
+ name = grp.getgrgid(_st.st_gid).gr_name
+ except Exception: # noqa: BLE001 -- no group database, or no entry for this gid
+ name = ""
+ # A name that cannot be pasted on as one group is treated as no name at all, so the
+ # node is reported by its GID rather than prescribed for. Above every branch that
+ # reads the name, including the privileged one, which compares whole strings and so
+ # is exactly what a name carrying a comma walks through.
+ if name and not _group_name_is_prescribable(name):
+ name = ""
+ # Asked of the GID, and BEFORE the name is required, because the lookup above is
+ # exactly what fails in a minimal container: with no entry for gid 0 it raised, the
+ # node was filed as an ordinary unnamed GID, and the repair became `groupadd -g 0`
+ # plus a usermod into the root group -- the grant this branch exists to refuse.
+ # gid 0 is the root group whether or not the database names it.
+ if _st.st_gid == 0:
+ _root = name or "root"
+ if _root not in privileged:
+ privileged.append(_root)
+ continue
+ # Joining one of these would open the node and hand over a great deal else with
+ # it, so a device node owned by one is a udev misconfiguration to report rather
+ # than a membership to prescribe. An unnamed GID cannot match, so this may sit
+ # above the naming branches and keep the shell half's single ordering.
+ if name in _PRIVILEGED_GROUPS:
+ if name not in privileged:
+ privileged.append(name)
+ continue
+ # os.access already said the node is shut, so if this account is in the owning
+ # group the group bits are not what is denying it: a container device cgroup or an
+ # LSM is. usermod would exit 0 and leave the node exactly as closed. ABOVE the
+ # unnamed branch as well, since `groupadd -g` plus `--group-add` is the same empty
+ # promise for a numeric owner this account already carries.
+ if _st.st_gid in _mine:
+ _held = name or str(_st.st_gid)
+ if _held not in already:
+ already.append(_held)
+ continue
+ if not name:
+ if _st.st_gid not in unnamed:
+ unnamed.append(_st.st_gid)
+ continue
+ if name not in joinable:
+ joinable.append(name)
+ return joinable, unnamed, no_group, acl, owned, privileged, already, external
+
+
+_RENDER_NODE_GLOB = "/dev/dri/renderD*"
+
+
+def _amd_nodes_the_runtime_lacks(*, needs_kfd: bool = True) -> "list[str]":
+ """The AMD device nodes this backend opens that do not exist at all.
+
+ Gated on AMD evidence, so this cannot fire on a host with no AMD card -- the trap a bare
+ "no render node" test would fall into, since every vendor's nodes live under the same
+ glob. The KFD topology is the first source: world-readable sysfs that names the vendor.
+
+ A container given --device /dev/dri and not --device /dev/kfd commonly masks
+ /sys/class/kfd as well, and there the topology proves nothing while DRM still names an
+ AMD render node outright. That host is exactly the one a HIP caller cannot run on, so it
+ falls back to the same confirmed-DRM evidence the closed-node path already trusts.
+ Confirmed, not assumed: _a_confirmed_amd_render_node_exists reads the vendor rather than
+ accepting an unreadable one, or an NVIDIA-only box could claim an AMD node.
+ """
+ if not _kfd_topology_has_an_amd_gpu() and not (
+ _kfd_topology_amd_state() is None and _a_confirmed_amd_render_node_exists()
+ ):
+ return []
+ lacks = []
+ if needs_kfd and not os.path.exists(_KFD_NODE):
+ lacks.append(_KFD_NODE)
+ if not _amd_render_node_exists():
+ lacks.append(_RENDER_NODE_GLOB)
+ return lacks
+
+
+def amd_closed_nodes_block_the_runtime(*, needs_kfd: bool = True) -> bool:
+ """Whether the closed nodes actually leave the runtime with no way in.
+
+ A closed node explains an empty GPU probe only when it is a node the runtime would
+ have used. On a multi-AMD host one render node can be shut while a sibling is open,
+ and ROCm then had /dev/kfd plus a render node and still enumerated nothing, so the
+ closed one is a second finding rather than the cause; a caller returning it as the
+ sole diagnosis sends the user after a repair that leaves the probe just as empty.
+
+ ``needs_kfd`` is the caller's backend: HIP opens /dev/kfd as well as a render node,
+ Vulkan only the render node. /dev/kfd has no sibling, so a closed one blocks outright.
+
+ True on a host this cannot read, which keeps the closed node as the stated reason and
+ is what the callers said before this existed.
+ """
+ # Missing is not open. A node the runtime needs and that does not exist leaves it with
+ # no way in exactly as a shut one does, and amd_node_permission_hint() names the repair
+ # for it -- so answering only about CLOSED nodes suppressed that hint at every caller,
+ # on the two container shapes where it is the whole diagnosis.
+ if _amd_nodes_the_runtime_lacks(needs_kfd = needs_kfd):
+ return True
+ closed = amd_nodes_closed_to_this_user()
+ if not closed:
+ return False
+ if needs_kfd and _KFD_NODE in closed:
+ return True
+ # The open sibling only answers for a runtime free to use it. A selector narrowing to
+ # particular GPUs may well have selected the CLOSED one, and this cannot tell which,
+ # so the sibling stops being evidence: fail closed, as this already does for a host it
+ # cannot read, rather than suppressing the repair for the node the run will use.
+ # HIP's selectors only, so only for a caller that goes through HIP. Vulkan reads none
+ # of these -- which is the whole reason needs_kfd exists -- so a Vulkan probe is still
+ # free to use the open sibling, and returning the permission hint as its sole cause
+ # would send a Vulkan failure after a group change that cannot empty-probe it.
+ if needs_kfd and _a_per_gpu_mask_narrows_the_runtime():
+ return True
+ return not an_amd_render_node_is_open()
+
+
+def _selector_exposes_every_gpu(
+ value: str,
+ count: "int | None",
+ *,
+ repeat_ends_the_list: bool = False,
+) -> bool:
+ """Whether every measured GPU survives this selector, so it excludes nothing.
+
+ HIP_VISIBLE_DEVICES=0,1 on a two-GPU host selects the whole host: the open sibling is
+ still reachable, and reading it as a narrowing hands that host the group repair in place
+ of the driver diagnosis it needs.
+
+ ``repeat_ends_the_list`` is ROCr's rule, not clr's: RvdFilter terminates on a token that
+ "maps to a device that has been previously selected", so ROCR_VISIBLE_DEVICES=0,0,1
+ surfaces ONE device, where clr stops only on a token that is not its own index written
+ back out and leaves both. An unmappable token TERMINATES the list rather than discarding
+ what came before it, clr having already pushed every device it accepted, so
+ HIP_VISIBLE_DEVICES=0,1,-1 exposes both. The prefix decides this, and an unreadable
+ count answers False: unmeasured goes on meaning narrowed.
+ """
+ if not count:
+ return False
+ seen = set()
+ for token in value.split(","):
+ token = token.strip()
+ try:
+ index = int(token)
+ except ValueError:
+ break
+ # clr's own rule: the token has to be the index written back out.
+ if str(index) != token or index < 0 or index >= count:
+ break
+ if index in seen and repeat_ends_the_list:
+ break
+ seen.add(index)
+ return len(seen) == count
+
+
+def _a_per_gpu_mask_narrows_the_runtime() -> bool:
+ """Whether a selector narrows the runtime away from some of this host's AMD GPUs.
+
+ Read for one purpose only: an OPEN render node is an alternative way in only when the
+ runtime is free to use it. Nothing here maps a render node back to the index a mask
+ selected it by, so under a NARROWING mask the open node may belong to a GPU the mask
+ excludes and stops being evidence.
+
+ Read the way the runtime layers them, which is not "all four at once". ROCr is its own
+ layer. The HIP layer then reads HIP_VISIBLE_DEVICES when it is non-empty and
+ CUDA_VISIBLE_DEVICES otherwise, so a CUDA value under a HIP one is SHADOWED and narrows
+ nothing -- the same precedence _explain_empty_gpu_probe's _hip_layer_var applies.
+ GPU_DEVICE_ORDINAL is OpenCL's, and independent of both.
+ """
+ count = amd_kfd_gpu_node_count()
+ _hip_layer = (
+ "HIP_VISIBLE_DEVICES"
+ if os.environ.get("HIP_VISIBLE_DEVICES", "").strip()
+ else "CUDA_VISIBLE_DEVICES"
+ )
+ for _name in (
+ "ROCR_VISIBLE_DEVICES",
+ _hip_layer,
+ # ROCm's fourth visibility variable, modelled elsewhere in this tree
+ # (tests/test_amd_smi_inventory_matches_hip.py, llama_cpp.py's own selector
+ # check). Omitting it left one of the four selectors crediting a sibling the
+ # runtime had been narrowed away from.
+ "GPU_DEVICE_ORDINAL",
+ ):
+ _value = os.environ.get(_name, "").strip()
+ if not _value:
+ continue
+ if not _selector_exposes_every_gpu(
+ _value,
+ count,
+ # ROCr terminates its list on a repeat; clr does not.
+ repeat_ends_the_list = _name == "ROCR_VISIBLE_DEVICES",
+ ):
+ return True
+ return False
+
+
+def _shell_word(value: str) -> str:
+ """``value`` as a single shell word, for a command the user is going to paste.
+
+ NSS names are not identifiers: winbind hands back DOMAIN\\user, and a group name may
+ carry whitespace or a metacharacter, so interpolating one raw lets the shell de-escape,
+ split or expand it -- and usermod then names an account that is not the one holding the
+ node shut, or runs something nobody typed under the sudo the line already carries.
+ shlex.quote leaves an ordinary name exactly as it was, so the common command is
+ unchanged; install.sh's _shell_quote is the same safe set for the same reason.
+ """
+ if value == "$USER":
+ # The placeholder the no-passwd fallback emits on a platform with no pwd module,
+ # and the one value here that is meant to be expanded rather than named.
+ return value
+ return shlex.quote(value)
+
+
+def _repair_account() -> Optional[str]:
+ """The account the usermod commands must name, or None when there is no such account.
+
+ os.getuid() is who os.access answered for above. USER and LOGNAME are inherited, so a
+ container that changes its numeric user without resetting them names somebody else, and
+ following the hint then modifies an account that is not the one holding the device shut.
+
+ None rather than a guess. A uid with no passwd entry is the ordinary shape of `docker
+ run --user 1234`, and there USER commonly still says root: usermod would then succeed,
+ change an identity nothing is running as, and leave the nodes exactly as shut. The
+ callers print the container-level repair instead, which is the one that works there.
+ """
+ try:
+ import pwd
+ return pwd.getpwuid(os.getuid()).pw_name
+ except (KeyError, OSError):
+ return None
+ except (ImportError, AttributeError):
+ # No pwd module at all, which is Windows, where none of these nodes exist.
+ return os.environ.get("USER") or os.environ.get("LOGNAME") or "$USER"
+
+
+def amd_node_permission_hint(*, needs_kfd: bool = True) -> Optional[str]:
+ """One sentence naming the closed nodes and the command that opens them, or None.
+
+ Kept beside the probe so the capability message, the llama.cpp log and the
+ installer all say the same thing, and so a caller that only needs the yes/no does
+ not build a string.
+
+ The two nodes do not block the same backends. A closed render node stops every
+ one of them, since HIP and the Vulkan loader both open it. ``/dev/kfd`` stops only
+ HIP: Vulkan never opens it, so ``needs_kfd = False`` is how a Vulkan-only caller
+ says that a closed KFD node is not its problem, and answering otherwise would send
+ a Vulkan failure with some other cause after the wrong repair.
+ """
+ closed = amd_nodes_closed_to_this_user()
+ if not needs_kfd:
+ closed = [path for path in closed if path != _KFD_NODE]
+ # A node the runtime needs and that does not exist blocks exactly as hard as one it
+ # cannot open, and needs saying whether or not anything is closed: a container given
+ # --device /dev/kfd and not --device /dev/dri opens the one node it has and enumerates
+ # nothing, and the mirror image (only /dev/dri) leaves HIP with no /dev/kfd. Both used
+ # to be reachable only after a closed node had already produced a sentence.
+ missing = _amd_nodes_the_runtime_lacks(needs_kfd = needs_kfd)
+ parts: "list[str]" = []
+ trailing_command = ""
+ if closed:
+ # Claim only what the closed set actually blocks, and only for the devices it is
+ # about. A render node shut beside an OPEN sibling leaves both runtimes a complete
+ # path to that other GPU, so "no GPU backend can use the AMD card" is false there --
+ # and _explain_empty_gpu_probe reaches exactly that host, appending this sentence
+ # after saying the closed node is not why the probe is empty.
+ user = _repair_account()
+ joinable, unnamed, no_group, acl, owned, privileged, already, external = _groups_that_own(
+ closed
+ )
+ if not any(_p != _KFD_NODE for _p in closed):
+ _claim = "so ROCm cannot use the AMD card even though the driver is loaded"
+ elif an_amd_render_node_is_open():
+ _claim = (
+ "so no GPU backend can use the card behind them, even though the driver is "
+ "loaded and another AMD render node on this host is open"
+ )
+ else:
+ _claim = "so no GPU backend can use the AMD card even though the driver is loaded"
+ parts.append(f"This account cannot open {', '.join(closed)}, {_claim}.")
+ # Prescribed only where joining a group is the repair. A host whose nodes could not
+ # be stat'd at all still gets the documented pair, since some advice beats none; a
+ # host whose nodes were read and offer no joinable group gets the sentences below
+ # instead of a command that would fail.
+ if joinable or not (
+ unnamed or no_group or acl or owned or privileged or already or external
+ ):
+ groups = joinable or ["render", "video"]
+ joined = ",".join(groups)
+ plural = "group" if len(groups) == 1 else "groups"
+ if user is None:
+ _joins = " ".join(f"--group-add {_shell_word(_g)}" for _g in groups)
+ parts.append(
+ f"This uid has no entry in the passwd database, so usermod has no "
+ f"account to name: recreate the container passing {_joins}, or run it "
+ f"as an account this system knows."
+ )
+ else:
+ # Held back to the END of the message rather than appended in place. It
+ # is the one sentence that terminates in a command a user pastes, so
+ # anything joined after it ran straight on, rendering as
+ # "-a -G render ada ROCm needs /dev/kfd ...". Ending the message with it
+ # keeps the command copyable as written, where a full stop would be
+ # selected along with the account name.
+ trailing_command = (
+ f"Add the account to the {joined} {plural} and then log out and back "
+ f"in: sudo usermod -a -G {_shell_word(joined)} {_shell_word(user)}"
+ )
+ if unnamed:
+ _gids = ", ".join(str(_g) for _g in unnamed)
+ # One flag per GID: docker's --group-add takes a single value, so naming only
+ # the first leaves every other node shut on a host whose nodes differ in group.
+ _adds = " ".join(f"--group-add {_g}" for _g in unnamed)
+ _noun = "GID" if len(unnamed) == 1 else "GIDs"
+ _verb = "which has" if len(unnamed) == 1 else "which have"
+ # Creating the group only gives the numeric owner a name; the account is
+ # still outside it and the node is still shut. Both halves of the bare-host
+ # repair, and one per GID: with two unnamed GIDs a singular instruction repairs
+ # at most one of the nodes.
+ _each = "it" if len(unnamed) == 1 else "each of them"
+ _pairs = "; ".join(
+ f"sudo groupadd -g {_g} amdgpu{_g} && "
+ f"sudo usermod -a -G amdgpu{_g} {_shell_word(user)}"
+ for _g in unnamed
+ )
+ # A pair per GID: one groupadd names one numeric owner, so a host whose nodes
+ # differ in group had every node after the first left shut. The name is
+ # generated, not a <name> placeholder, since angle brackets are redirection
+ # operators and these are commands to paste. The && is load bearing: with an
+ # amdgpu<GID> group already at a different GID, an unchained usermod would
+ # SUCCEED against the wrong group and report success on a still-shut node.
+ if user is None:
+ # The bare-host half needs an account to add and there is none, so the
+ # container half is the whole repair for this shape.
+ parts.append(
+ f"Some of those nodes belong to {_noun} {_gids}, {_verb} no group entry "
+ f"on this system, and this uid has no passwd entry either, so neither "
+ f"groupadd nor usermod has anything to name: recreate the container "
+ f"passing {_adds}."
+ )
+ else:
+ parts.append(
+ f"Some of those nodes belong to {_noun} {_gids}, {_verb} no group entry "
+ f"on this system, so usermod cannot name them: create a group for "
+ f"{_each}, add the account to it and then log out and back in "
+ f"({_pairs}), or recreate the container passing {_adds}."
+ )
+ if no_group:
+ parts.append(
+ f"{', '.join(no_group)} does not grant its own group read and write, so no "
+ f"membership opens it: fix the udev rule or the node's permissions."
+ )
+ if owned:
+ parts.append(
+ f"{', '.join(owned)} is owned by this account, and POSIX stops at the owner "
+ f"bits once the uid matches, so no group membership opens it however its "
+ f"group bits read: fix the mode with chmod, or the udev rule that set it."
+ )
+ if already:
+ parts.append(
+ f"This account is already in the {', '.join(already)} "
+ f"{'group' if len(already) == 1 else 'groups'} that own those nodes, so "
+ f"usermod would change nothing: something outside the file mode is denying "
+ f"them, typically a container device cgroup or an LSM such as SELinux or "
+ f"AppArmor."
+ )
+ if external:
+ # Worded by the class that APPLIES rather than by the owner one. This bucket
+ # holds two shapes: a node this account owns whose owner bits grant rw, and one
+ # it neither owns nor shares a group with whose other bits do. Naming the owner
+ # for both told the second kind it owns a node it does not, which is a claim the
+ # reader can check and find false, in the one sentence whose job is to say the
+ # mode is not the problem.
+ parts.append(
+ f"{', '.join(external)} is already granted read and write by the permission "
+ f"bits that apply to this account, so the mode is not what is shutting it: "
+ f"something outside the file mode is denying it, typically a container "
+ f"device cgroup or an LSM such as SELinux or AppArmor."
+ )
+ if privileged:
+ parts.append(
+ f"Those nodes belong to the {', '.join(privileged)} group, which grants a "
+ f"great deal besides the GPU, so joining it is not the repair: fix the udev "
+ f"rule so the node is owned by render or video instead."
+ )
+ if acl:
+ parts.append(
+ # Every path, not acl[0]: the sentence lists them all, ROCm needs all of
+ # them, and their ACLs need not agree, so checking only the first can leave
+ # the second blocker undiagnosed. getfacl takes several paths.
+ f"{', '.join(acl)} carries a POSIX ACL, so the group permissions cannot be "
+ f"read from its mode: check the real grant with getfacl {' '.join(acl)} "
+ f"before changing group membership."
+ )
+ # Group membership cannot create a device node, so these stand whether or not anything
+ # above was said. install.sh says the same two things; this is the runtime half.
+ if _KFD_NODE in missing:
+ # Two wordings, because only one of the two ways in here proves anything about the
+ # driver. A topology that NAMES an AMD GPU is the amdkfd driver's own sysfs, so the
+ # stack is loaded and reinstalling it changes nothing. Reached instead off a
+ # confirmed DRM render node with the topology masked, that claim is unsupported --
+ # the node may be missing because the stack is not loaded -- so the sentence says
+ # what is true of both: the mapping is what to fix first.
+ if _kfd_topology_amd_state() is True:
+ parts.append(
+ "ROCm needs /dev/kfd, which is not present here, but the KFD topology "
+ "already names an AMD GPU, so the kernel driver is loaded and reinstalling "
+ "ROCm changes nothing: the node itself is missing. Under Docker, recreate "
+ "the container with --device /dev/kfd --device /dev/dri; on a bare host it "
+ "is a udev or devtmpfs problem. No group membership creates it."
+ )
+ else:
+ parts.append(
+ "ROCm needs /dev/kfd, which is not present here, while an AMD render node "
+ "is. Under Docker, recreate the container with --device /dev/kfd --device "
+ "/dev/dri; on a bare host, check that the amdgpu kernel module is loaded. "
+ "No group membership creates it."
+ )
+ if _RENDER_NODE_GLOB in missing:
+ # The mapping this caller needs, not both nodes always: Vulkan never opens /dev/kfd,
+ # so naming it here hands a container another host device for nothing.
+ _devices = "--device /dev/kfd --device /dev/dri" if needs_kfd else "--device /dev/dri"
+ parts.append(
+ f"No AMD render node (/dev/dri/renderD*) is present, and ROCm and Vulkan both "
+ f"open one, so the device mapping needs fixing; under Docker that is "
+ f"{_devices}."
+ )
+ if trailing_command:
+ parts.append(trailing_command)
+ return " ".join(parts) or None
diff --git a/studio/backend/utils/hardware/hardware.py b/studio/backend/utils/hardware/hardware.py
index 762b3473822..0ac4c600def 100644
--- a/studio/backend/utils/hardware/hardware.py
+++ b/studio/backend/utils/hardware/hardware.py
@@ -931,6 +931,38 @@ def _torch_reports_a_hip_runtime() -> bool:
return False
+def _torch_reports_another_vendors_runtime() -> bool:
+ """Whether the installed torch is a CUDA or XPU build, whatever its label says.
+
+ The mirror of _torch_reports_a_hip_runtime, and needed for the same reason in reverse.
+ A conda or locally built CUDA wheel carries no +cu tag, so the label names no vendor,
+ and the intent fallback then reads a stale recorded ROCm flavor as "this wheel targets
+ AMD" -- on a host whose real repair is reinstalling ROCm torch. torch.version.cuda is
+ written by the build itself and settles it.
+
+ An import failure is answered from disk, as _torch_reports_a_hip_runtime and
+ _torch_reports_an_xpu_runtime already answer it: torch/version.py records the runtime
+ whether or not the package imports. Returning False there made the clearing above inert
+ on the path it exists for, letting a stale ROCm flavor speak for a CUDA or XPU wheel.
+ """
+ if TORCH_IMPORT_ERROR is not None:
+ # A ROCm build records hip and may record cuda besides.
+ if _torch_reports_a_hip_runtime():
+ return False
+ _markers = _installed_torch_markers_on_disk()
+ return bool(_markers["cuda"]) or bool(_markers["xpu"])
+ try:
+ import torch
+
+ _version = getattr(torch, "version", None)
+ # A ROCm build sets hip and can carry cuda besides, so hip is read first.
+ if getattr(_version, "hip", None) is not None:
+ return False
+ return bool(getattr(_version, "cuda", None)) or bool(getattr(_version, "xpu", None))
+ except Exception:
+ return False
+
+
# Marketing name -> gfx, mirroring setup.ps1's $nameArchTable and install_python_stack._WIN_GPU_NAME_ARCH_TABLE. Only names those two route to a wheel family, since this decides whether a repair could change anything. Most specific first.
_GPU_NAME_GFX_TABLE: "list[tuple[str, str]]" = [
(r"9070|9080|R9700", "gfx1201"),
@@ -1563,6 +1595,11 @@ def current_chat_only_verdict() -> tuple[Optional[str], Optional[str]]:
return (reason, detail) if frozen_but_measurable else ("no_gpu", None)
+# A wheel label naming another vendor's accelerator: +cu128, +xpu. Matched on the local
+# part so a version like 2.9.0 can never look like one.
+_WHEEL_LABEL_OTHER_VENDOR_RE = re.compile(r"\+[a-z]*(?:cu\d|xpu)")
+
+
def _gpu_present_but_unusable_message(
feature: str, verdict: Optional[tuple[Optional[str], Optional[str]]] = None
) -> Optional[str]:
@@ -1572,18 +1609,62 @@ def _gpu_present_but_unusable_message(
if reason not in ("torch_cpu_build", "torch_cuda_unavailable"):
return None
installed = f" (installed {detail})" if detail else ""
+ # No reinstall changes group membership (#10466), so a closed node is not a mismatch.
+ # Recording AMD is necessary but not sufficient: a hybrid host on CUDA torch records
+ # both vendors, and there the verdict is about the NVIDIA card.
+ vendors = {str(vendor).lower() for vendor in CHAT_ONLY_MISMATCH_VENDORS}
+ # Worth MENTIONING is about the hardware; REPLACES the reinstall advice is about the
+ # wheel, and only a ROCm one is repaired by opening a node. Conflating them was the bug.
+ _label = (detail or "").lower()
+ # Intent is the LAST resort: it outlives the wheel, so a venv that recorded ROCm and
+ # then had CUDA installed over it still answers yes. Label and live runtime describe
+ # what is installed NOW; a bare +cpu names no vendor, so intent still gets its say.
+ wheel_targets_amd = (
+ "rocm" in _label
+ or "hip" in _label
+ or _torch_reports_a_hip_runtime()
+ or (
+ not _WHEEL_LABEL_OTHER_VENDOR_RE.search(_label)
+ # An untagged CUDA build (conda, or local) names no vendor, so the regex
+ # clears it and stale intent would speak for a wheel that is not AMD's.
+ and not _torch_reports_another_vendors_runtime()
+ and _expected_rocm_flavor_was_chosen()
+ )
+ )
+ amd_is_the_target = vendors == {"amd"} or wheel_targets_amd
+ node_hint = None
+ if "amd" in vendors and amd_is_the_target:
+ try:
+ from utils.hardware.amd import (
+ amd_closed_nodes_block_the_runtime,
+ amd_node_permission_hint,
+ )
+
+ # Only when the closed set leaves the runtime no way in: an open sibling
+ # render node means ROCm had a complete path and failed anyway.
+ if amd_closed_nodes_block_the_runtime():
+ node_hint = amd_node_permission_hint()
+ except Exception:
+ node_hint = None
+ # Replaces the reinstall advice only for a ROCm wheel, which the closed node fully
+ # explains. A CPU-only or other-vendor wheel needs BOTH repairs and is given both.
+ if node_hint and reason == "torch_cuda_unavailable" and wheel_targets_amd:
+ return f"This host has a GPU, but {feature} cannot use it. {node_hint}"
# Both routes, always. The repair row exists only in the desktop app and only for a backend it manages, so a browser-hosted Studio, or a desktop attached to a server someone started from a terminal, was being sent to a control that is not on the page.
if reason == "torch_cpu_build":
return (
f"This host has a GPU, but the installed PyTorch is a CPU-only build{installed}, "
f"so {feature} cannot use it. Reinstall the GPU build: use Repair installation "
f"in Settings in the desktop app, or re-run the Unsloth installer."
+ + (f" {node_hint}" if node_hint else "")
)
return (
f"This host has a GPU, but the installed PyTorch{installed} cannot initialise it, so "
f"{feature} cannot use it. This is usually a driver or runtime mismatch; reinstalling "
f"a matching PyTorch build fixes it. Use Repair installation in Settings in the "
- f"desktop app, or re-run the Unsloth installer."
+ f"desktop app, or re-run the Unsloth installer." + (f" {node_hint}" if node_hint else "")
+ # Appended, not substituted: the wheel is still the repair, but the node is
+ # still closed and the matching ROCm build will need it.
)
diff --git a/studio/install_llama_prebuilt.py b/studio/install_llama_prebuilt.py
index ac247ffd9cd..4ec71458413 100644
--- a/studio/install_llama_prebuilt.py
+++ b/studio/install_llama_prebuilt.py
@@ -9419,8 +9419,15 @@ def _vulkan_loader_allows(path: str) -> bool:
manifest's basename. A manifest filtered out is registered and never loaded, so
counting it hands an integrated host a Vulkan build with no AMD device.
- Disable is read before select precisely so "disable everything, then name one back"
- works, hence select answering alone when it is set.
+ Select is an allowlist and disable is a denylist that WINS over it, rather than being
+ skipped when select is set. The loader documents the order the other way round from the
+ way this once read it: "The values from the disable environment variable will be
+ considered before the enable or select environment variable", and
+ VK_LOADER_DRIVERS_DISABLE is "also checked before other driver environment variables
+ (such as VK_LOADER_DRIVERS_SELECT)" (Vulkan-Loader, LoaderInterfaceArchitecture.md).
+ Drivers have no VK_LOADER_LAYERS_ALLOW counterpart to name one back, so selecting
+ radeon* while also disabling radeon* leaves the real loader with no driver, where this
+ counted Radeon as usable and reported only the device-node repair.
"""
def _globs(env_name: str) -> list[str]:
@@ -9428,11 +9435,11 @@ def _vulkan_loader_allows(path: str) -> bool:
return [entry.strip() for entry in value.split(",") if entry.strip()]
name = PurePath(path).name
- select = _globs("VK_LOADER_DRIVERS_SELECT")
- if select:
- return any(_vulkan_glob_matches(pattern, name) for pattern in select)
disable = _globs("VK_LOADER_DRIVERS_DISABLE")
- return not any(_vulkan_glob_matches(pattern, name) for pattern in disable)
+ if any(_vulkan_glob_matches(pattern, name) for pattern in disable):
+ return False
+ select = _globs("VK_LOADER_DRIVERS_SELECT")
+ return any(_vulkan_glob_matches(pattern, name) for pattern in select) if select else True
# Per call, not at import: Path.home() raises with no USERPROFILE.
diff --git a/tests/studio/install/test_rocm_support.py b/tests/studio/install/test_rocm_support.py
index 756cfafd790..e9a76b0d3f0 100644
--- a/tests/studio/install/test_rocm_support.py
+++ b/tests/studio/install/test_rocm_support.py
@@ -76,8 +76,10 @@ def _extract_sh_function_body(source: str, name: str) -> str:
"""Return a shell function body from `source` by brace matching."""
needle = f"{name}() {{"
start = source.find(needle)
- if start < 0:
- return ""
+ # Raise rather than return "". Every caller names a function that install.sh defines,
+ # so a miss is a rename, and an empty body makes each caller fail on its own subject
+ # -- a vendor_id check reported as missing when the vendor_id check is still there.
+ assert start >= 0, f"install.sh defines no {name}()"
depth = 0
i = start + len(needle) - 1 # land on the opening brace
n = len(source)
@@ -2457,12 +2459,15 @@ class TestHasRocmGpuKfdVendorGuard:
), "_has_rocm_gpu must skip gpu_id 0 nodes (CPU nodes)"
def test_install_sh_has_vendor_check(self):
- """_has_amd_rocm_gpu in install.sh sysfs fallback must also check vendor_id 4098."""
+ """The install.sh sysfs fallback must also check vendor_id 4098.
+
+ The probe and the NVIDIA veto live in one function: tests/sh lifts helpers out of
+ install.sh one at a time by name, so a wrapper over a private helper leaves those
+ harnesses calling something undefined.
+ """
sh_path = PACKAGE_ROOT / "install.sh"
source = sh_path.read_text(encoding = "utf-8")
- func_start = source.find("_has_amd_rocm_gpu()")
- func_end = source.find("\n}", func_start)
- func_body = source[func_start:func_end]
+ func_body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
assert "vendor_id" in func_body, "_has_amd_rocm_gpu sysfs fallback must check vendor_id"
assert "4098" in func_body, "_has_amd_rocm_gpu must require AMD vendor_id 4098 (0x1002)"
@@ -3124,12 +3129,10 @@ class TestInstallShStructure:
assert "export UNSLOTH_TORCH_BACKEND" in source
def test_kfd_sysfs_amd_vendor_check_in_has_amd_rocm_gpu(self):
- """_has_amd_rocm_gpu sysfs fallback must require AMD vendor_id 4098 (nvidia-open registers KFD nodes too)."""
+ """The sysfs fallback must require AMD vendor_id 4098 (nvidia-open registers KFD nodes too)."""
sh_path = PACKAGE_ROOT / "install.sh"
source = sh_path.read_text(encoding = "utf-8")
- func_start = source.find("_has_amd_rocm_gpu()")
- func_end = source.find("\n}", func_start)
- func_body = source[func_start:func_end]
+ func_body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
assert (
"vendor_id" in func_body
), "_has_amd_rocm_gpu sysfs fallback must check vendor_id to exclude NVIDIA KFD nodes"
@@ -3150,9 +3153,7 @@ class TestInstallShStructure:
"""
sh_path = PACKAGE_ROOT / "install.sh"
source = sh_path.read_text(encoding = "utf-8")
- func_start = source.find("_has_amd_rocm_gpu()")
- func_end = source.find("\n}", func_start)
- func_body = source[func_start:func_end]
+ func_body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
assert "$2 == 4098" in func_body, (
"_has_amd_rocm_gpu KFD awk must match `vendor_id 4098` as a single-line "
"condition so no per-node state can leak across KFD nodes"
@@ -8019,6 +8020,24 @@ if __name__ == "__main__":
pytest.main([__file__, "-v"])
+def test_the_shell_extractor_says_so_when_a_function_is_gone():
+ """A rename must fail as a rename. Three checks here read the ROCm probe with find()
+ and an empty body on a miss, so folding _amd_rocm_gpu_visible back into
+ _has_amd_rocm_gpu made all three report a missing vendor_id check that was still
+ there. The helper now refuses the name instead."""
+ source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
+ with pytest.raises(AssertionError, match = "_amd_rocm_gpu_visible"):
+ _extract_sh_function_body(source, "_amd_rocm_gpu_visible")
+
+
+def test_the_extractor_still_returns_the_probe_it_does_define():
+ """The control: the surviving name must still come back with a body, or the check
+ above would pass on a helper that refuses everything."""
+ source = (PACKAGE_ROOT / "install.sh").read_text(encoding = "utf-8")
+ body = _extract_sh_function_body(source, "_has_amd_rocm_gpu")
+ assert body.startswith("_has_amd_rocm_gpu() {") and "vendor_id" in body
+
+
# ─────────────────────────────────────────────────────────────────────────────
# TEST: install_python_stack.py -- the AMD bitsandbytes reinstall is evidence-gated