ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
fix(llama-server): keep the CUDA kernel cache across recreates (#8280)
Mike Bradleybdbfd498b741
diff --git a/ods/Makefile b/ods/Makefileindex 343ff395b..bc5ee290f 100644--- a/ods/Makefile+++ b/ods/Makefile@@ -187,6 +187,7 @@ test: ## Run unit and contract tests@bash tests/test-comfyui-user-dir-persisted.sh@echo "=== ComfyUI CUDA kernel cache is persisted ==="@bash tests/test-comfyui-cuda-cache-persisted.sh+ @bash tests/test-llama-cuda-cache-persisted.sh@echo ""@echo "=== macOS ods config show secret-mask ==="@bash tests/test-macos-config-show-secret-mask.shdiff --git a/ods/docker-compose.nvidia.yml b/ods/docker-compose.nvidia.ymlindex 3bcf60863..a70508699 100644--- a/ods/docker-compose.nvidia.yml+++ b/ods/docker-compose.nvidia.yml@@ -14,6 +14,16 @@ services:# per-model LLAMA_ARG_SPEC_TYPE (e.g. draft-mtp) wins. Set# LLAMA_SPEC_TYPE=none in .env to turn it off.- LLAMA_ARG_SPEC_TYPE=${LLAMA_ARG_SPEC_TYPE:-${LLAMA_SPEC_TYPE:-ngram-mod}}+ # The image has native kernels for compute capability 8.6, 8.9 and 12.0a+ # only. On Hopper, datacenter Blackwell and GB10 the driver compiles the+ # rest at load and on first use, and in the container layer that repeated+ # after every recreate: on Workstation06's GB300 (2026-10-27), 25 s to+ # ready and 42 s for the first long prompt, against 3 s and 1.3 s with+ # this cache. Elsewhere nothing is compiled and the folder stays empty.+ - CUDA_CACHE_PATH=/cuda-cache+ - CUDA_CACHE_MAXSIZE=4294967296+ volumes:+ - ./data/llama-server/cuda-cache:/cuda-cache:zdeploy:resources:reservations:diff --git a/ods/installers/phases/06-directories.sh b/ods/installers/phases/06-directories.shindex cefa24b60..0b60e3f61 100755--- a/ods/installers/phases/06-directories.sh+++ b/ods/installers/phases/06-directories.sh@@ -564,6 +564,8 @@ elseods_progress 38 "directories" "Creating directory structure"mkdir -p "$INSTALL_DIR"/{config,data,models}mkdir -p "$INSTALL_DIR"/data/{open-webui,whisper,tts,n8n,qdrant,models,privacy-shield,ape,token-spy,hermes,persona}+ # llama-server's CUDA kernel cache (docker-compose.nvidia.yml).+ mkdir -p "$INSTALL_DIR"/data/llama-server/cuda-cache# The host agent writes extension progress as the install owner. Created# here, the Dashboard API (UID 1000, which may be another user) cannot# make it first and lock the owner's host agent out of it.diff --git a/ods/lib/backup-paths.sh b/ods/lib/backup-paths.shindex 1c16422d1..095153304 100644--- a/ods/lib/backup-paths.sh+++ b/ods/lib/backup-paths.sh@@ -59,6 +59,7 @@ ODS_BACKUP_EXCLUDED_DATA_PATHS=("data/models" # GGUF weights — re-downloadable, tens of GB"data/whisper" # STT model cache"data/embeddings" # TEI model cache+ "data/llama-server" # CUDA kernel cache, root-owned; rebuilt on the next start# dashboard-api runtime files: progress, status and receipts that the# running stack rewrites, or copies of state captured elsewhere."data/extensions-library" # catalog copy; re-copied by the installerdiff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txtindex 6c1b964fa..7f9e30789 100644--- a/ods/tests/ci-suite.txt+++ b/ods/tests/ci-suite.txt@@ -61,6 +61,7 @@ tests/test-cli-stt-env.pytests/test-cli-update-verification.shtests/test-comfyui-user-dir-persisted.shtests/test-comfyui-cuda-cache-persisted.sh+tests/test-llama-cuda-cache-persisted.shtests/test_comfyui_extension_contract.pytests/test_extension_assets.pytests/test-compose-failure-evidence.shdiff --git a/ods/tests/test-llama-cuda-cache-persisted.sh b/ods/tests/test-llama-cuda-cache-persisted.shnew file mode 100755index 000000000..98c970154--- /dev/null+++ b/ods/tests/test-llama-cuda-cache-persisted.sh@@ -0,0 +1,40 @@+#!/usr/bin/env bash+# The pinned llama.cpp CUDA image has native kernels for compute capability+# 8.6, 8.9 and 12.0a only. On Hopper, datacenter Blackwell and GB10 the CUDA+# driver compiles the rest at load and on first use and caches the result. In+# the container layer that cache was lost on every recreate: on Workstation06's+# GB300 (2026-10-27) each restart took 25 s to ready and 42 s for the first long+# prompt, against 3 s and 1.3 s with the cache kept. This asserts the cache is+# bind-mounted read-write, sized for those GPUs, and created by the installer.+set -euo pipefail++ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"+COMPOSE="${ODS_NVIDIA_COMPOSE_UNDER_TEST:-$ROOT_DIR/docker-compose.nvidia.yml}"+PHASE="${ODS_PHASE06_UNDER_TEST:-$ROOT_DIR/installers/phases/06-directories.sh}"++fail() { echo "[FAIL] $*" >&2; exit 1; }+pass() { echo "[PASS] $*"; }++for f in "$COMPOSE" "$PHASE"; do [[ -f "$f" ]] || fail "missing $f"; done++# 1. The cache is a read-write bind mount on llama-server, and CUDA is pointed at it.+service="$(awk '/^ llama-server:/{f=1; next} f && /^ [a-z]/{f=0} f' "$COMPOSE")"+mount_line="$(grep -E '^\s*-\s*\./data/llama-server/cuda-cache:' <<<"$service" || true)"+[[ -n "$mount_line" ]] \+ || fail "docker-compose.nvidia.yml does not bind-mount ./data/llama-server/cuda-cache into llama-server; compiled kernels stay in the container layer"+grep -qE ':ro(,|$|[[:space:]])' <<<"$mount_line" \+ && fail "the CUDA cache is mounted read-only: $mount_line"+target="$(sed -E 's/^\s*-\s*\.\/data\/llama-server\/cuda-cache:([^:]+).*/\1/' <<<"$mount_line")"+grep -qE "^\s*-\s*CUDA_CACHE_PATH=${target}\s*$" <<<"$service" \+ || fail "llama-server's CUDA_CACHE_PATH does not name the mounted cache folder $target"+pass "llama-server's CUDA cache is bind-mounted read-write at CUDA_CACHE_PATH ($target)"++# 2. The driver's 1 GiB default is too small for a full compiled kernel set.+grep -qE '^\s*-\s*CUDA_CACHE_MAXSIZE=4294967296\s*$' <<<"$service" \+ || fail "llama-server's CUDA_CACHE_MAXSIZE is not raised to the driver's 4 GiB maximum"+pass "llama-server's CUDA cache may grow to 4 GiB"++# 3. The installer creates the folder with the other core data folders.+grep -qE '^\s*mkdir -p "\$INSTALL_DIR"/data/llama-server/cuda-cache\s*$' "$PHASE" \+ || fail "phase 06 does not create data/llama-server/cuda-cache"+pass "the installer creates llama-server's CUDA cache folder"