Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

2.2 KB
Raw Blame
# ODS — NVIDIA GPU Overlay (Core Services Only)
# ComfyUI GPU config moved to extensions/services/comfyui/compose.nvidia.yaml
# Whisper GPU config moved to extensions/services/whisper/compose.nvidia.yaml
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.nvidia.yml up -d

services:
  llama-server:
    image: ${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-cuda-b11429@sha256:883915ea20a4b350e98f7284098f4d400c21e9b366e248d397cd42d90602a84c}
    environment:
      # n-gram speculative decoding. The target model verifies every drafted
      # token, so output is the target model's own. It needs llama.cpp b8955+
      # (dedicated ngram-mod params) and b8842+ (checkpoints for hybrid models
      # such as Qwen3.5); the pinned b11429 image has both. An explicit
      # per-model LLAMA_ARG_SPEC_TYPE (e.g. draft-mtp) wins. Set
      # LLAMA_SPEC_TYPE=none in .env to turn it off.
      - LLAMA_ARG_SPEC_TYPE=${LLAMA_ARG_SPEC_TYPE:-${LLAMA_SPEC_TYPE:-ngram-mod}}
      # The image has native kernels for compute capability 8.6, 8.9 and 12.0a
      # only. On Hopper, datacenter Blackwell and GB10 the driver compiles the
      # rest at load and on first use, and in the container layer that repeated
      # after every recreate: on Workstation06's GB300 (2026-10-27), 25 s to
      # ready and 42 s for the first long prompt, against 3 s and 1.3 s with
      # this cache. Elsewhere nothing is compiled and the folder stays empty.
      - CUDA_CACHE_PATH=/cuda-cache
      - CUDA_CACHE_MAXSIZE=4294967296
    volumes:
      - ./data/llama-server/cuda-cache:/cuda-cache:z
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all
              capabilities: [gpu]
        limits:
          cpus: '${LLAMA_CPU_LIMIT:-16.0}'
          memory: ${LLAMA_SERVER_MEMORY_LIMIT:-64G}

  dashboard-api:
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              count: all  # nvidia-smi needs visibility into all GPUs for monitoring
              capabilities: [utility]

  open-webui:
    environment:
      AUDIO_STT_MODEL: "${AUDIO_STT_MODEL:-deepdml/faster-whisper-large-v3-turbo-ct2}"
      ENABLE_IMAGE_GENERATION: "${ENABLE_IMAGE_GENERATION:-true}"