Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

4.5 KB
Raw Blame
# ODS - vLLM engine overlay
#
# Used when .env sets LLM_BACKEND=vllm: vLLM serves the model on one big
# NVIDIA GPU (datacenter Blackwell, compute capability 10.x, or the RTX PRO
# 6000 class, 12.0) instead of the in-stack llama-server. It preallocates
# VLLM_GPU_MEMORY_UTILIZATION of that GPU, the catalog model's own share
# when it sets one. As on the host-native route, llama-server is off and
# model-router stays; endpoints.json points model-router at vllm-server:8000.
#
# This file is intentionally not standalone. scripts/resolve-compose-stack.sh
# layers it right after docker-compose.nvidia.yml when LLM_BACKEND=vllm, and
# refuses it together with a host-native, external or cloud model route or a
# non-NVIDIA GPU. The installer writes the VLLM_* values from the model
# catalog; without VLLM_MODEL_DIR, VLLM_SERVED_MODEL or VLLM_GPU_UUID,
# Compose stops and names the missing one.
#
# Validate the resolved stack with:
#
#   LLM_BACKEND=vllm ./scripts/resolve-compose-stack.sh --ods-mode local --gpu-backend nvidia --tier NV_ULTRA

services:
  llama-server:
    profiles:
      - local-inference
    restart: "no"

  vllm-server:
    # Digest-pinned multi-arch image (config/dependency-lock.json). The
    # installer writes the same reference to VLLM_IMAGE.
    image: ${VLLM_IMAGE:-vllm/vllm-openai:v0.32.0@sha256:befccf0e7e4089aa23926bb6f887a85c0d46c2a99e6de5c5ffdf5cb7957c5b34}
    container_name: ods-vllm-server
    restart: unless-stopped
    # Reached only on the ODS network (model-router); no host port.
    expose:
      - "8000"
    security_opt:
      - no-new-privileges:true
    shm_size: 16g
    environment:
      # The first start loads 130-200 GB of weights and compiles kernels;
      # vLLM's shorter default would give up first.
      - VLLM_ENGINE_READY_TIMEOUT_S=3600
      # The weights are local. Never reach the Hugging Face Hub.
      - HF_HUB_OFFLINE=1
      # vLLM sends anonymous usage reports to its developers unless told not to.
      - VLLM_NO_USAGE_STATS=1
      - DO_NOT_TRACK=1
      # The CUDA driver's kernel cache, kept in the mounted cache folder. In
      # the container layer it was lost on every recreate (see llama-server
      # in docker-compose.nvidia.yml); 4 GiB is the driver's maximum.
      - CUDA_CACHE_PATH=/root/.cache/nv/ComputeCache
      - CUDA_CACHE_MAXSIZE=4294967296
      # The other kernel caches default outside /root/.cache (~/.tilelang,
      # ~/.triton, /tmp/torchinductor_root), so every recreate rebuilt them:
      # GLM-5.3-Flash JIT-compiles 136 TileLang kernels on each start.
      - TILELANG_CACHE_DIR=/root/.cache/tilelang
      - TRITON_CACHE_DIR=/root/.cache/triton
      - TORCHINDUCTOR_CACHE_DIR=/root/.cache/torchinductor
    volumes:
      # The model's snapshot folder under data/models, read-only.
      - ./data/models/${VLLM_MODEL_DIR:?VLLM_MODEL_DIR must name the model folder under data/models}:/model:ro,z
      # FlashInfer autotune, torch.compile and CUDA kernel caches. Without
      # this mount every recreate (ods update, a restart on a new model)
      # compiles again. Rebuilt on start, so backups skip it.
      - ./data/vllm/cache:/root/.cache:z
    # Compose splits this string like a shell, so VLLM_ARGS (the catalog
    # entry's argument string) becomes separate arguments to `vllm serve`.
    command: >-
      --model /model
      --served-model-name ${VLLM_SERVED_MODEL:?VLLM_SERVED_MODEL must name the model id vLLM serves}
      --host 0.0.0.0
      --port 8000
      --gpu-memory-utilization ${VLLM_GPU_MEMORY_UTILIZATION:-0.90}
      ${VLLM_ARGS:-}
    deploy:
      resources:
        reservations:
          devices:
            # Exactly the GPU the installer picked. Do not add a second entry
            # in another overlay: Compose appends reservation lists, and the
            # container would get an extra GPU.
            - driver: nvidia
              device_ids: ["${VLLM_GPU_UUID:?VLLM_GPU_UUID must name the GPU vLLM owns}"]
              capabilities: [gpu]
    healthcheck:
      # /health answers 200 with an empty body once the engine is ready.
      # The image has python3 because vLLM runs on it; curl may be missing.
      test: ["CMD", "python3", "-c", "import sys,urllib.request;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8000/health',timeout=5).status==200 else 1)"]
      interval: 30s
      timeout: 10s
      retries: 5
      # As long as VLLM_ENGINE_READY_TIMEOUT_S, so Docker does not call a
      # first start unhealthy before vLLM itself gives up.
      start_period: 60m
    logging:
      driver: "json-file"
      options:
        max-size: "10m"
        max-file: "3"