Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(comfyui): keep the CUDA kernel cache across container recreates (#8260)

Mike Bradleya29fa7f19963
diff --git a/ods/Makefile b/ods/Makefile
index 7cc3d969a..343ff395b 100644
--- a/ods/Makefile
+++ b/ods/Makefile
@@ -185,6 +185,8 @@ test: ## Run unit and contract tests
@bash tests/test-validate-manifest-schema.sh
@echo "=== ComfyUI user directory is persisted ==="
@bash tests/test-comfyui-user-dir-persisted.sh
+ @echo "=== ComfyUI CUDA kernel cache is persisted ==="
+ @bash tests/test-comfyui-cuda-cache-persisted.sh
@echo ""
@echo "=== macOS ods config show secret-mask ==="
@bash tests/test-macos-config-show-secret-mask.sh
diff --git a/ods/extensions/services/comfyui/compose.nvidia.yaml b/ods/extensions/services/comfyui/compose.nvidia.yaml
index 819c62699..d4ac0ccad 100644
--- a/ods/extensions/services/comfyui/compose.nvidia.yaml
+++ b/ods/extensions/services/comfyui/compose.nvidia.yaml
@@ -17,6 +17,15 @@ services:
# Saved workflows and UI settings. Without this they live only in the
# container layer and are lost on every recreate (e.g. `ods update`).
- ./data/comfyui/user:/user:z
+ # The CUDA driver's kernel cache. On a GPU this PyTorch build has no
+ # kernels for (GB300, sm_103) the driver compiles ~850 MB of them for the
+ # first image. In the container layer that work was lost on every
+ # recreate, so the first image after an update took 52 s instead of 7 s.
+ - ./data/comfyui/cuda-cache:/cuda-cache:z
+ environment:
+ CUDA_CACHE_PATH: /cuda-cache
+ # The 1 GiB default barely holds one sm_103 set; 4 GiB is the maximum.
+ CUDA_CACHE_MAXSIZE: "4294967296"
shm_size: '8g'
deploy:
resources:
diff --git a/ods/installers/phases/11-services.sh b/ods/installers/phases/11-services.sh
index 74e4895ae..3c6326e9c 100755
--- a/ods/installers/phases/11-services.sh
+++ b/ods/installers/phases/11-services.sh
@@ -1138,9 +1138,9 @@ else
fi
# NVIDIA ComfyUI also needs output/input/workflows bind-mount dirs
if [[ "$GPU_BACKEND" == "nvidia" ]]; then
- mkdir -p "$INSTALL_DIR/data/comfyui"/{output,input,workflows,user}
+ mkdir -p "$INSTALL_DIR/data/comfyui"/{output,input,workflows,user,cuda-cache}
if ! _phase11_prepare_uid1000_bind_data \
- "$INSTALL_DIR/data/comfyui" models output input user; then
+ "$INSTALL_DIR/data/comfyui" models output input user cuda-cache; then
ai_bad "Could not prepare NVIDIA ComfyUI data for its container user."
exit 1
fi
diff --git a/ods/lib/rootless-ownership.sh b/ods/lib/rootless-ownership.sh
index 3e8420e42..3121a3012 100644
--- a/ods/lib/rootless-ownership.sh
+++ b/ods/lib/rootless-ownership.sh
@@ -479,7 +479,7 @@ ods_prepare_comfyui_data_ownership() {
[[ "$host_uid" != 1000 ]] || return 0
host_gid=$(id -g) || return 1
[[ "$host_gid" =~ ^[0-9]+$ ]] || return 1
- for path in models output input user; do
+ for path in models output input user cuda-cache; do
[[ -d "$install_dir/data/comfyui/$path" && ! -L "$install_dir/data/comfyui/$path" ]] || {
echo "[error] The ComfyUI $path folder is not a real directory." >&2
return 1
diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txt
index 6cf72d6e8..546649009 100644
--- a/ods/tests/ci-suite.txt
+++ b/ods/tests/ci-suite.txt
@@ -60,6 +60,7 @@ tests/test-cli-status-optional-webui.sh
tests/test-cli-stt-env.py
tests/test-cli-update-verification.sh
tests/test-comfyui-user-dir-persisted.sh
+tests/test-comfyui-cuda-cache-persisted.sh
tests/test_comfyui_extension_contract.py
tests/test_extension_assets.py
tests/test-compose-failure-evidence.sh
diff --git a/ods/tests/test-comfyui-cuda-cache-persisted.sh b/ods/tests/test-comfyui-cuda-cache-persisted.sh
new file mode 100755
index 000000000..0f5fba96c
--- /dev/null
+++ b/ods/tests/test-comfyui-cuda-cache-persisted.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# On a GPU the image's PyTorch build has no kernels for (GB300, sm_103), the
+# CUDA driver compiles them on first use and caches the result. In the
+# container layer that cache was lost on every recreate, so the first image
+# after each update took 52 s instead of 7 s (Workstation06, 2026-10-27). This
+# asserts the cache is bind-mounted, sized for that GPU, and created and
+# prepared for the container user like the other NVIDIA ComfyUI folders.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+COMPOSE="${ODS_COMFYUI_COMPOSE_UNDER_TEST:-$ROOT_DIR/extensions/services/comfyui/compose.nvidia.yaml}"
+PHASE="${ODS_PHASE11_UNDER_TEST:-$ROOT_DIR/installers/phases/11-services.sh}"
+
+fail() { echo "[FAIL] $*" >&2; exit 1; }
+pass() { echo "[PASS] $*"; }
+
+for f in "$COMPOSE" "$PHASE"; do [[ -f "$f" ]] || fail "missing $f"; done
+
+# 1. The cache is a read-write bind mount, and CUDA is pointed at it.
+mount_line="$(grep -E '^\s*-\s*\./data/comfyui/cuda-cache:' "$COMPOSE" || true)"
+[[ -n "$mount_line" ]] \
+ || fail "compose.nvidia.yaml does not bind-mount ./data/comfyui/cuda-cache; compiled kernels stay in the container layer"
+grep -qE ':ro(,|$|[[:space:]])' <<<"$mount_line" \
+ && fail "the CUDA cache is mounted read-only: $mount_line"
+target="$(sed -E 's/^\s*-\s*\.\/data\/comfyui\/cuda-cache:([^:]+).*/\1/' <<<"$mount_line")"
+grep -qE "^\s*CUDA_CACHE_PATH:\s*\"?${target}\"?\s*$" "$COMPOSE" \
+ || fail "CUDA_CACHE_PATH does not name the mounted cache folder $target"
+pass "the CUDA cache is bind-mounted read-write at CUDA_CACHE_PATH ($target)"
+
+# 2. One sm_103 kernel set nearly fills the driver's 1 GiB default.
+grep -qE '^\s*CUDA_CACHE_MAXSIZE:\s*"4294967296"\s*$' "$COMPOSE" \
+ || fail "CUDA_CACHE_MAXSIZE is not raised to the driver's 4 GiB maximum"
+pass "the CUDA cache may grow to 4 GiB"
+
+# 3. The installer creates it and prepares it for UID 1000 with the rest.
+grep -qE 'mkdir -p "\$INSTALL_DIR/data/comfyui"/\{[^}]*\bcuda-cache\b[^}]*\}' "$PHASE" \
+ || fail "phase 11 does not create data/comfyui/cuda-cache, so Docker would auto-create it as root"
+grep -qE '"\$INSTALL_DIR/data/comfyui" models output input user cuda-cache; then' "$PHASE" \
+ || fail "phase 11 does not prepare data/comfyui/cuda-cache for the container's UID 1000"
+pass "the installer creates the CUDA cache and prepares it for UID 1000"
diff --git a/ods/tests/test-rootless-docker-ownership.sh b/ods/tests/test-rootless-docker-ownership.sh
index ccd3997f3..f63b440ca 100644
--- a/ods/tests/test-rootless-docker-ownership.sh
+++ b/ods/tests/test-rootless-docker-ownership.sh
@@ -273,7 +273,7 @@ pass "rootless APE add-back keeps its namespace repair; other services are refus
# Workstation06 (2026-10-26): the install owner is UID 1001 and ComfyUI was
# added from the Dashboard, so Phase 11 never prepared its UID 1000 folders.
-mkdir -p "$INSTALL_DIR/data/comfyui"/{models,output,input,user}
+mkdir -p "$INSTALL_DIR/data/comfyui"/{models,output,input,user,cuda-cache}
: > "$CALLS"
(
source "$LIB"
@@ -287,7 +287,7 @@ mkdir -p "$INSTALL_DIR/data/comfyui"/{models,output,input,user}
docker() { printf 'docker %s\n' "$*" >> "$CALLS"; }
ods_prepare_comfyui_data_ownership "$INSTALL_DIR" nvidia
) || fail "rootful ComfyUI add-back preparation failed: $(cat "$CALLS")"
-for path in models output input user; do
+for path in models output input user cuda-cache; do
grep -qx "fix data/comfyui/$path|1000:4242|ods-comfyui|775" "$CALLS" \
|| fail "rootful ComfyUI add-back did not prepare data/comfyui/$path for UID 1000: $(cat "$CALLS")"
grep -q "^docker run .* -v $INSTALL_DIR/data/comfyui/$path:/data .* chmod -R ug+rwX /data\$" "$CALLS" \