ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
# ODS — CPU-Only Overlay (Linux / no GPU)
# Uses llama.cpp server CPU build. No NVIDIA/AMD/Intel GPU required.
# For low-RAM systems, set LLAMA_SERVER_MEMORY_LIMIT=4G in .env
#
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.cpu.yml up -d
services:
llama-server:
# b8248 ignores LLAMA_ARG_REASONING from the base service; b9014 honors it.
image: ${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-b9014@sha256:2e7953dfef88f302bf0683bffa7dc1f8d86ef75910380bc41126ec5b8bedaf53}
command:
- --model
- /models/${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
# Same served id as docker-compose.base.yml: the GGUF file name.
- --alias
- ${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
- --host
- 0.0.0.0
- --port
- "8080"
- --n-gpu-layers
- "0"
- --ctx-size
- "${CTX_SIZE:-8192}"
- --batch-size
- "${LLAMA_BATCH_SIZE:-512}"
- --threads
- "${LLAMA_THREADS:-4}"
- --parallel
- "${LLAMA_PARALLEL:-1}"
- --metrics
environment:
# n-gram speculative decoding; same contract as docker-compose.nvidia.yml.
# The pinned b9014 image has the b8955+ ngram-mod implementation. Set
# LLAMA_SPEC_TYPE=none in .env to turn it off.
- LLAMA_ARG_SPEC_TYPE=${LLAMA_ARG_SPEC_TYPE:-${LLAMA_SPEC_TYPE:-ngram-mod}}
deploy:
resources:
limits:
cpus: '${LLAMA_CPU_LIMIT:-8.0}'
memory: ${LLAMA_SERVER_MEMORY_LIMIT:-6G}
reservations:
cpus: '${LLAMA_CPU_RESERVATION:-1.0}'
memory: 2G