Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

fix(llama-server): keep the CUDA kernel cache across recreates (#8280)

Mike Bradleybdbfd498b741
diff --git a/ods/Makefile b/ods/Makefile
index 343ff395b..bc5ee290f 100644
--- a/ods/Makefile
+++ b/ods/Makefile
@@ -187,6 +187,7 @@ test: ## Run unit and contract tests
@bash tests/test-comfyui-user-dir-persisted.sh
@echo "=== ComfyUI CUDA kernel cache is persisted ==="
@bash tests/test-comfyui-cuda-cache-persisted.sh
+ @bash tests/test-llama-cuda-cache-persisted.sh
@echo ""
@echo "=== macOS ods config show secret-mask ==="
@bash tests/test-macos-config-show-secret-mask.sh
diff --git a/ods/docker-compose.nvidia.yml b/ods/docker-compose.nvidia.yml
index 3bcf60863..a70508699 100644
--- a/ods/docker-compose.nvidia.yml
+++ b/ods/docker-compose.nvidia.yml
@@ -14,6 +14,16 @@ services:
# per-model LLAMA_ARG_SPEC_TYPE (e.g. draft-mtp) wins. Set
# LLAMA_SPEC_TYPE=none in .env to turn it off.
- LLAMA_ARG_SPEC_TYPE=${LLAMA_ARG_SPEC_TYPE:-${LLAMA_SPEC_TYPE:-ngram-mod}}
+ # The image has native kernels for compute capability 8.6, 8.9 and 12.0a
+ # only. On Hopper, datacenter Blackwell and GB10 the driver compiles the
+ # rest at load and on first use, and in the container layer that repeated
+ # after every recreate: on Workstation06's GB300 (2026-10-27), 25 s to
+ # ready and 42 s for the first long prompt, against 3 s and 1.3 s with
+ # this cache. Elsewhere nothing is compiled and the folder stays empty.
+ - CUDA_CACHE_PATH=/cuda-cache
+ - CUDA_CACHE_MAXSIZE=4294967296
+ volumes:
+ - ./data/llama-server/cuda-cache:/cuda-cache:z
deploy:
resources:
reservations:
diff --git a/ods/installers/phases/06-directories.sh b/ods/installers/phases/06-directories.sh
index cefa24b60..0b60e3f61 100755
--- a/ods/installers/phases/06-directories.sh
+++ b/ods/installers/phases/06-directories.sh
@@ -564,6 +564,8 @@ else
ods_progress 38 "directories" "Creating directory structure"
mkdir -p "$INSTALL_DIR"/{config,data,models}
mkdir -p "$INSTALL_DIR"/data/{open-webui,whisper,tts,n8n,qdrant,models,privacy-shield,ape,token-spy,hermes,persona}
+ # llama-server's CUDA kernel cache (docker-compose.nvidia.yml).
+ mkdir -p "$INSTALL_DIR"/data/llama-server/cuda-cache
# The host agent writes extension progress as the install owner. Created
# here, the Dashboard API (UID 1000, which may be another user) cannot
# make it first and lock the owner's host agent out of it.
diff --git a/ods/lib/backup-paths.sh b/ods/lib/backup-paths.sh
index 1c16422d1..095153304 100644
--- a/ods/lib/backup-paths.sh
+++ b/ods/lib/backup-paths.sh
@@ -59,6 +59,7 @@ ODS_BACKUP_EXCLUDED_DATA_PATHS=(
"data/models" # GGUF weights — re-downloadable, tens of GB
"data/whisper" # STT model cache
"data/embeddings" # TEI model cache
+ "data/llama-server" # CUDA kernel cache, root-owned; rebuilt on the next start
# dashboard-api runtime files: progress, status and receipts that the
# running stack rewrites, or copies of state captured elsewhere.
"data/extensions-library" # catalog copy; re-copied by the installer
diff --git a/ods/tests/ci-suite.txt b/ods/tests/ci-suite.txt
index 6c1b964fa..7f9e30789 100644
--- a/ods/tests/ci-suite.txt
+++ b/ods/tests/ci-suite.txt
@@ -61,6 +61,7 @@ tests/test-cli-stt-env.py
tests/test-cli-update-verification.sh
tests/test-comfyui-user-dir-persisted.sh
tests/test-comfyui-cuda-cache-persisted.sh
+tests/test-llama-cuda-cache-persisted.sh
tests/test_comfyui_extension_contract.py
tests/test_extension_assets.py
tests/test-compose-failure-evidence.sh
diff --git a/ods/tests/test-llama-cuda-cache-persisted.sh b/ods/tests/test-llama-cuda-cache-persisted.sh
new file mode 100755
index 000000000..98c970154
--- /dev/null
+++ b/ods/tests/test-llama-cuda-cache-persisted.sh
@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+# The pinned llama.cpp CUDA image has native kernels for compute capability
+# 8.6, 8.9 and 12.0a only. On Hopper, datacenter Blackwell and GB10 the CUDA
+# driver compiles the rest at load and on first use and caches the result. In
+# the container layer that cache was lost on every recreate: on Workstation06's
+# GB300 (2026-10-27) each restart took 25 s to ready and 42 s for the first long
+# prompt, against 3 s and 1.3 s with the cache kept. This asserts the cache is
+# bind-mounted read-write, sized for those GPUs, and created by the installer.
+set -euo pipefail
+
+ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
+COMPOSE="${ODS_NVIDIA_COMPOSE_UNDER_TEST:-$ROOT_DIR/docker-compose.nvidia.yml}"
+PHASE="${ODS_PHASE06_UNDER_TEST:-$ROOT_DIR/installers/phases/06-directories.sh}"
+
+fail() { echo "[FAIL] $*" >&2; exit 1; }
+pass() { echo "[PASS] $*"; }
+
+for f in "$COMPOSE" "$PHASE"; do [[ -f "$f" ]] || fail "missing $f"; done
+
+# 1. The cache is a read-write bind mount on llama-server, and CUDA is pointed at it.
+service="$(awk '/^ llama-server:/{f=1; next} f && /^ [a-z]/{f=0} f' "$COMPOSE")"
+mount_line="$(grep -E '^\s*-\s*\./data/llama-server/cuda-cache:' <<<"$service" || true)"
+[[ -n "$mount_line" ]] \
+ || fail "docker-compose.nvidia.yml does not bind-mount ./data/llama-server/cuda-cache into llama-server; compiled kernels stay in the container layer"
+grep -qE ':ro(,|$|[[:space:]])' <<<"$mount_line" \
+ && fail "the CUDA cache is mounted read-only: $mount_line"
+target="$(sed -E 's/^\s*-\s*\.\/data\/llama-server\/cuda-cache:([^:]+).*/\1/' <<<"$mount_line")"
+grep -qE "^\s*-\s*CUDA_CACHE_PATH=${target}\s*$" <<<"$service" \
+ || fail "llama-server's CUDA_CACHE_PATH does not name the mounted cache folder $target"
+pass "llama-server's CUDA cache is bind-mounted read-write at CUDA_CACHE_PATH ($target)"
+
+# 2. The driver's 1 GiB default is too small for a full compiled kernel set.
+grep -qE '^\s*-\s*CUDA_CACHE_MAXSIZE=4294967296\s*$' <<<"$service" \
+ || fail "llama-server's CUDA_CACHE_MAXSIZE is not raised to the driver's 4 GiB maximum"
+pass "llama-server's CUDA cache may grow to 4 GiB"
+
+# 3. The installer creates the folder with the other core data folders.
+grep -qE '^\s*mkdir -p "\$INSTALL_DIR"/data/llama-server/cuda-cache\s*$' "$PHASE" \
+ || fail "phase 06 does not create data/llama-server/cuda-cache"
+pass "the installer creates llama-server's CUDA cache folder"