Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Docker toolkit installer: stop pipefail turning a grep match into a miss (#10402)

Daniel Hana6e9880c4591
diff --git a/docker/install_nvidia_toolkit.sh b/docker/install_nvidia_toolkit.sh
index efdc41717f5..b76a0774b70 100755
--- a/docker/install_nvidia_toolkit.sh
+++ b/docker/install_nvidia_toolkit.sh
@@ -18,8 +18,15 @@ fail() { printf 'ERROR: %s\n' "$1" >&2; exit "${2:-1}"; }
command -v docker >/dev/null 2>&1 \
|| fail "docker is not installed. Install Docker Engine first (https://docs.docker.com/engine/install/), then run this again." 2
+# Every grep of a command's output below reads a variable, not a pipe. `grep -q` exits on the first
+# match and closes the pipe; `set -o pipefail` above then promotes the producer's SIGPIPE (141) to the
+# status of the whole pipeline, so a match reads as a miss whenever the producer was still writing.
+# Measured on `docker info` with a 50ms pause mid-output: status 141, the Docker Desktop guard skipped,
+# and the script elevating to install on the one daemon it exists to leave alone.
+
# Docker Desktop ships its own GPU integration; installing here would configure a daemon it does not use. Checked before elevating.
-if docker info 2>/dev/null | grep -qi 'Operating System: Docker Desktop'; then
+docker_info="$(docker info 2>/dev/null || true)"
+if grep -qi 'Operating System: Docker Desktop' <<<"$docker_info"; then
if [[ "$(uname -s)" == Darwin ]]; then
say "Docker Desktop on macOS: no NVIDIA GPU can be attached on a Mac, so there is nothing to install."
say "The image runs CPU-only there: drop --gpus and set UNSLOTH_ALLOW_CPU=1."
@@ -48,10 +55,13 @@ esac
# Rootless Docker keeps its own daemon, which the steps below would miss; root sees a different one, and `curl | sudo bash` arrives root with DOCKER_HOST stripped, so probe SUDO_UID's socket too.
rootless_daemon() {
- docker info --format '{{join .SecurityOptions ","}}' 2>/dev/null | grep -q rootless && return 0
- [[ -n "${SUDO_UID:-}" && -S "${RUN_USER_DIR}/${SUDO_UID}/docker.sock" ]] \
- && DOCKER_HOST="unix://${RUN_USER_DIR}/${SUDO_UID}/docker.sock" \
- docker info --format '{{join .SecurityOptions ","}}' 2>/dev/null | grep -q rootless
+ local opts
+ opts="$(docker info --format '{{join .SecurityOptions ","}}' 2>/dev/null || true)"
+ grep -q rootless <<<"$opts" && return 0
+ [[ -n "${SUDO_UID:-}" && -S "${RUN_USER_DIR}/${SUDO_UID}/docker.sock" ]] || return 1
+ opts="$(DOCKER_HOST="unix://${RUN_USER_DIR}/${SUDO_UID}/docker.sock" \
+ docker info --format '{{join .SecurityOptions ","}}' 2>/dev/null || true)"
+ grep -q rootless <<<"$opts"
}
if rootless_daemon; then
fail "rootless Docker detected. Follow NVIDIA's rootless procedure instead:
@@ -70,7 +80,9 @@ fi
# On WSL 2 the Windows driver puts nvidia-smi under /usr/lib/wsl/lib, which sudo's secure_path drops from PATH.
NVSMI="$(command -v nvidia-smi 2>/dev/null || true)"
[[ -z "$NVSMI" && -x "${WSL_LIB_DIR}/nvidia-smi" ]] && NVSMI="${WSL_LIB_DIR}/nvidia-smi"
-if [[ -z "$NVSMI" ]] || ! "$NVSMI" -L 2>/dev/null | grep -q '^GPU'; then
+gpu_list=""
+[[ -n "$NVSMI" ]] && gpu_list="$("$NVSMI" -L 2>/dev/null || true)"
+if [[ -z "$NVSMI" ]] || ! grep -q '^GPU' <<<"$gpu_list"; then
if grep -qi microsoft "$PROC_VERSION" 2>/dev/null; then
fail "no NVIDIA GPU is visible in this WSL 2 distro. Install a current NVIDIA Windows driver
from nvidia.com (never a Linux driver inside WSL), restart WSL (wsl --shutdown), then run
@@ -82,7 +94,10 @@ if [[ -z "$NVSMI" ]] || ! "$NVSMI" -L 2>/dev/null | grep -q '^GPU'; then
Unsloth images need driver 570.26 or newer." 2
fi
MIN_DRIVER=570.26
-DRIVER="$("$NVSMI" --query-gpu=driver_version --format=csv,noheader 2>/dev/null | head -1 | tr -d '[:space:]')"
+# `sed -n 1p`, not `head -1`: head closes the pipe after one line, and nvidia-smi prints one per GPU,
+# so on a multi-GPU host the SIGPIPE becomes the substitution's status and `set -e` kills the script
+# here with nothing printed. sed reads to the end. (Measured: exit 141, no output, 8 GPUs.)
+DRIVER="$("$NVSMI" --query-gpu=driver_version --format=csv,noheader 2>/dev/null | sed -n 1p | tr -d '[:space:]')"
driver_ok() {
[[ -n "$DRIVER" ]] && [[ "$(printf '%s\n' "$MIN_DRIVER" "$DRIVER" | sort -V | head -1)" == "$MIN_DRIVER" ]]
}
@@ -95,7 +110,8 @@ case "$(uname -m)" in
esac
configured() {
- docker info 2>/dev/null | grep -qi 'Runtimes:.*nvidia'
+ local info; info="$(docker info 2>/dev/null || true)"
+ grep -qi 'Runtimes:.*nvidia' <<<"$info"
}
if configured; then
diff --git a/tests/python/test_docker_nvidia_toolkit_install.py b/tests/python/test_docker_nvidia_toolkit_install.py
index 03690ab2c1d..d35f7d3a39c 100644
--- a/tests/python/test_docker_nvidia_toolkit_install.py
+++ b/tests/python/test_docker_nvidia_toolkit_install.py
@@ -51,6 +51,9 @@ _REAL_TOOLS = (
"cut",
"rm",
"true",
+ # A stub that pauses mid-output is how the SIGPIPE cases below are made
+ # deterministic; without it they reproduce only on a loaded runner.
+ "sleep",
)
@@ -83,6 +86,8 @@ def _setup(
wsl: bool = False,
driver_version: str = "580.65.06",
rootless: bool | str = False,
+ chunked: bool = False,
+ gpus: int = 1,
) -> tuple[Path, Path, dict]:
bindir = tmp_path / "bin"
bindir.mkdir()
@@ -95,10 +100,14 @@ def _setup(
rec = f'echo "$(basename "$0") $*" >> {log}\n'
_stub(bindir / "id", f"echo {uid}\n")
_stub(bindir / "sudo", rec + "exit 0\n")
+ # One driver line per GPU, as the real one prints. `pause` between them is what makes an
+ # early-exiting consumer (head -1) close the pipe while this is still writing.
+ pause = "sleep 0.05\n" if chunked else ""
+ query = "".join(f'echo " {driver_version}"\n{pause}' for _ in range(gpus)) + "exit 0\n"
_stub(
bindir / "nvidia-smi",
(
- f'if [ "$1" = --query-gpu=driver_version ]; then echo " {driver_version}"; exit 0; fi\n'
+ f'if [ "$1" = --query-gpu=driver_version ]; then\n{query}fi\n'
'echo "GPU 0: NVIDIA H100 (UUID: GPU-abc)"\n'
)
if driver
@@ -119,6 +128,9 @@ def _setup(
+ 'if [ "$1" = context ]; then case "${DOCKER_CONTEXT:-}" in remote*) echo "tcp://gpu-box:2376" ;; missing*) echo "context not found" >&2; exit 1 ;; *) echo "unix:///var/run/docker.sock" ;; esac; exit 0; fi\n'
+ 'if [ "$1" = info ]; then\n'
+ (' echo " Operating System: Docker Desktop"\n' if desktop else "")
+ # A real daemon does not hand `docker info` over in one write. Pausing here puts the
+ # rest of the output after the point where a `grep -q` consumer has already matched.
+ + (" sleep 0.05\n" if chunked else "")
+ (
' if [ "$2" = "--format" ]; then\n'
' case "${DOCKER_HOST:-}" in *docker.sock) echo "name=rootless,name=seccomp" ;; *) echo "name=seccomp" ;; esac; exit 0\n'
@@ -131,6 +143,8 @@ def _setup(
)
)
+ f' if [ -e {marker} ]; then echo " Runtimes: io.containerd.runc.v2 nvidia runc"; else echo " Runtimes: io.containerd.runc.v2 runc"; fi\n'
+ # Trailing output, so a consumer that stops at the Runtimes line leaves this unwritten.
+ + (' sleep 0.05\n echo " Default Runtime: runc"\n' if chunked else "")
+ " exit 0\nfi\n"
+ (
"exit 0\n"
@@ -484,6 +498,44 @@ def test_the_default_sockets_are_accepted(tmp_path: Path):
assert "not the system daemon" not in res.stderr, sock
+def test_docker_desktop_is_still_seen_when_the_daemon_answers_in_pieces(tmp_path: Path):
+ """The guard must not depend on winning a race with its own consumer.
+
+ `grep -q` exits on the first match and closes the pipe, and under `set -o pipefail` the
+ producer's SIGPIPE (141) becomes the status of the pipeline, so the match reads as a miss.
+ Whether that happens depends only on whether `docker info` still had output to write, which
+ is why it showed up as an intermittent CI failure (the script printed "Re-running with sudo."
+ and exited 0 where the test expected the Docker Desktop refusal) rather than a steady one.
+ """
+ _, log, env = _setup(tmp_path, desktop = True, driver = False, uid = 1000, chunked = True)
+ res = _run(env)
+ assert res.returncode == 2, res.stdout + res.stderr
+ assert "Docker Desktop for Linux has no NVIDIA GPU support" in res.stderr
+ # The point of checking before elevating: no sudo, and nothing installed.
+ assert not any(c.startswith(("sudo", "apt-get")) for c in _calls(log))
+
+
+def test_a_chunked_daemon_does_not_hide_the_nvidia_runtime(tmp_path: Path):
+ """Same pipeline shape in `configured()`, and the cost is a pointless reinstall."""
+ _, log, env = _setup(tmp_path, configured = True, chunked = True)
+ res = _run(env)
+ assert res.returncode == 0, res.stdout + res.stderr
+ assert "already lists the nvidia runtime" in res.stdout
+ assert not any(c.startswith(("apt-get", "nvidia-ctk")) for c in _calls(log))
+
+
+def test_the_driver_version_survives_a_multi_gpu_host(tmp_path: Path):
+ """`head -1` closes the pipe after one line and nvidia-smi prints one per GPU.
+
+ Inside a command substitution that SIGPIPE becomes the assignment's status, and `set -e`
+ then ends the script where it stands, with nothing printed for the user to act on.
+ """
+ _, _, env = _setup(tmp_path, chunked = True, gpus = 8)
+ res = _run(env)
+ assert res.returncode == 0, f"rc={res.returncode} out={res.stdout!r} err={res.stderr!r}"
+ assert res.stdout.strip(), "the script exited without printing anything"
+
+
def test_docker_desktop_on_macos_says_cpu_only(tmp_path: Path):
_, log, env = _setup(tmp_path, desktop = True, driver = False, uid = 1000)
_stub(tmp_path / "bin" / "uname", 'if [ "$1" = -s ]; then echo Darwin; else echo arm64; fi\n')