ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
# ODS — NVIDIA GPU Overlay (Core Services Only)
# ComfyUI GPU config moved to extensions/services/comfyui/compose.nvidia.yaml
# Whisper GPU config moved to extensions/services/whisper/compose.nvidia.yaml
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.nvidia.yml up -d
services:
llama-server:
image: ${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-cuda-b11429@sha256:883915ea20a4b350e98f7284098f4d400c21e9b366e248d397cd42d90602a84c}
environment:
# n-gram speculative decoding. The target model verifies every drafted
# token, so output is the target model's own. It needs llama.cpp b8955+
# (dedicated ngram-mod params) and b8842+ (checkpoints for hybrid models
# such as Qwen3.5); the pinned b11429 image has both. An explicit
# per-model LLAMA_ARG_SPEC_TYPE (e.g. draft-mtp) wins. Set
# LLAMA_SPEC_TYPE=none in .env to turn it off.
- LLAMA_ARG_SPEC_TYPE=${LLAMA_ARG_SPEC_TYPE:-${LLAMA_SPEC_TYPE:-ngram-mod}}
# The image has native kernels for compute capability 8.6, 8.9 and 12.0a
# only. On Hopper, datacenter Blackwell and GB10 the driver compiles the
# rest at load and on first use, and in the container layer that repeated
# after every recreate: on Workstation06's GB300 (2026-10-27), 25 s to
# ready and 42 s for the first long prompt, against 3 s and 1.3 s with
# this cache. Elsewhere nothing is compiled and the folder stays empty.
- CUDA_CACHE_PATH=/cuda-cache
- CUDA_CACHE_MAXSIZE=4294967296
volumes:
- ./data/llama-server/cuda-cache:/cuda-cache:z
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all
capabilities: [gpu]
limits:
cpus: '${LLAMA_CPU_LIMIT:-16.0}'
memory: ${LLAMA_SERVER_MEMORY_LIMIT:-64G}
dashboard-api:
deploy:
resources:
reservations:
devices:
- driver: nvidia
count: all # nvidia-smi needs visibility into all GPUs for monitoring
capabilities: [utility]
open-webui:
environment:
AUDIO_STT_MODEL: "${AUDIO_STT_MODEL:-deepdml/faster-whisper-large-v3-turbo-ct2}"
ENABLE_IMAGE_GENERATION: "${ENABLE_IMAGE_GENERATION:-true}"