Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

2.6 KB
Raw Blame
# ODS — Intel Arc GPU Overlay (SYCL backend)
# Requires: Intel Arc GPU with oneAPI Level Zero runtime on host
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.intel.yml up -d
#
# Supported hardware:
#   ARC tier      — Arc A770 16 GB  (and future Arc B-series ≥12 GB)
#   ARC_LITE tier — Arc A750 8 GB, A380 6 GB
#
# Host prerequisites:
#   apt install intel-opencl-icd intel-level-zero-gpu level-zero
#   usermod -aG video,render $USER   (then re-login)
#   # Verify: clinfo | grep -i "intel arc"

services:
  llama-server:
    image: ${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-intel-b9014@sha256:9c7bbaad3663523a3deb8927d3cfbf58d33f00a7634c69843e9eeeda01568c1b}
    devices:
      - /dev/dri:/dev/dri
    group_add:
      - "${VIDEO_GID:-44}"
      - "${RENDER_GID:-992}"
    environment:
      # Level Zero selects the Intel Arc GPU.
      # SYCL_CACHE_PERSISTENT is deliberately not set: with the oneAPI 2025.3
      # runtime in the b9014 image it crashes llama-server (ggml-org/llama.cpp
      # #21474, #22095; intel/llvm#22853). ONEAPI_DEVICE_SELECTOR can be set in
      # .env; use level_zero:0 on hosts with more than one Intel GPU
      # (ggml-org/llama.cpp#21747).
      - ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR:-level_zero:gpu}
      # Enable Intel GPU System Management Interface for telemetry
      - ZES_ENABLE_SYSMAN=1
    command:
      - --model
      - /models/${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
      # Same served id as docker-compose.base.yml: the GGUF file name.
      - --alias
      - ${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
      - --host
      - 0.0.0.0
      - --port
      - "8080"
      - --n-gpu-layers
      - "${N_GPU_LAYERS:-99}"
      - --ctx-size
      - "${CTX_SIZE:-32768}"
      # Compose replaces `command` wholesale rather than merging it, so the
      # documented runtime tunables have to be repeated here or they go inert
      # on this overlay.
      - --batch-size
      - "${LLAMA_BATCH_SIZE:-2048}"
      - --threads
      - "${LLAMA_THREADS:-4}"
      - --parallel
      - "${LLAMA_PARALLEL:-1}"
      - --metrics
    deploy:
      resources:
        limits:
          cpus: '${LLAMA_CPU_LIMIT:-16.0}'
          memory: ${LLAMA_SERVER_MEMORY_LIMIT:-24G}
        reservations:
          cpus: '${LLAMA_CPU_RESERVATION:-2.0}'
          memory: 4G

  dashboard-api:
    environment:
      # Hard-code sycl backend so the dashboard uses Intel sysfs GPU detection
      # regardless of .env state.
      - GPU_BACKEND=sycl
    volumes:
      - /sys/class/drm:/sys/class/drm:ro
      - /sys/class/hwmon:/sys/class/hwmon:ro