Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

3.6 KB
Raw Blame
# ODS — Intel Arc GPU Overlay (oneAPI SYCL, build-from-source)
#
# Builds llama-server from source using Intel oneAPI Base Toolkit.
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.arc.yml up -d
#
# Supported hardware:
#   ARC tier      — Arc A770 16 GB  (and future Arc B-series ≥12 GB)
#   ARC_LITE tier — Arc A750 8 GB, A380 6 GB
#
# Build arguments (customise via .env or shell export):
#   LLAMA_TAG       llama.cpp git tag (default: b9014)
#   LLAMA_COMMIT    commit LLAMA_TAG must resolve to (default: b9014's commit);
#                   set it together with LLAMA_TAG, or empty to skip the check
#   ONEAPI_VERSION  oneAPI Base Toolkit image tag (default: 2025.0.0-0-devel-ubuntu22.04)
#   LLAMA_ARC_IMAGE Override to a pre-built image and skip the local build entirely
#                   e.g. LLAMA_ARC_IMAGE=ghcr.io/ggml-org/llama.cpp:server-intel-b9014@sha256:9c7bbaad3663523a3deb8927d3cfbf58d33f00a7634c69843e9eeeda01568c1b
#
# First-run note:
#   The SYCL build compiles llama.cpp with Intel icx/icpx. Allow 10–20 min
#   on first `docker compose up --build`. Subsequent starts use Docker cache.
#
# Host prerequisites (Ubuntu/Debian):
#   apt install intel-opencl-icd intel-level-zero-gpu level-zero
#   usermod -aG video,render $USER   # re-login after
#   # Verify GPU: clinfo | grep -i "intel arc"

services:
  llama-server:
    build:
      context: ./images/llama-sycl
      dockerfile: Dockerfile
      args:
        LLAMA_TAG: ${LLAMA_TAG:-b9014}
        LLAMA_COMMIT: ${LLAMA_COMMIT-d4b0c22f9e67f0295e91dc1ab4f17c0fb2557fa4}
        ONEAPI_VERSION: ${ONEAPI_VERSION:-2025.0.0-0-devel-ubuntu22.04}
    # Set LLAMA_ARC_IMAGE in .env to skip the local build and pull a pre-built image.
    image: ${LLAMA_ARC_IMAGE:-ods-llama-sycl:local}
    devices:
      - /dev/dri:/dev/dri
    group_add:
      - "${VIDEO_GID:-44}"
      - "${RENDER_GID:-992}"
    environment:
      # Level Zero selects the Intel Arc GPU.
      # SYCL_CACHE_PERSISTENT is deliberately not set: with the oneAPI 2025.3
      # runtime in the b9014 image it crashes llama-server (ggml-org/llama.cpp
      # #21474, #22095; intel/llvm#22853). ONEAPI_DEVICE_SELECTOR can be set in
      # .env; use level_zero:0 on hosts with more than one Intel GPU
      # (ggml-org/llama.cpp#21747).
      - ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR:-level_zero:gpu}
      # Enable Intel GPU System Management Interface for telemetry
      - ZES_ENABLE_SYSMAN=1
    command:
      - --model
      - /models/${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
      # Same served id as docker-compose.base.yml: the GGUF file name.
      - --alias
      - ${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
      - --host
      - 0.0.0.0
      - --port
      - "8080"
      - --n-gpu-layers
      - "${N_GPU_LAYERS:-99}"
      - --ctx-size
      - "${CTX_SIZE:-32768}"
      # Compose replaces `command` wholesale rather than merging it, so the
      # documented runtime tunables have to be repeated here or they go inert
      # on this overlay.
      - --batch-size
      - "${LLAMA_BATCH_SIZE:-2048}"
      - --threads
      - "${LLAMA_THREADS:-4}"
      - --parallel
      - "${LLAMA_PARALLEL:-1}"
      - --metrics
    deploy:
      resources:
        limits:
          cpus: '${LLAMA_CPU_LIMIT:-16.0}'
          memory: ${LLAMA_SERVER_MEMORY_LIMIT:-24G}
        reservations:
          cpus: '${LLAMA_CPU_RESERVATION:-2.0}'
          memory: 4G

  dashboard-api:
    environment:
      # Hard-code sycl so the dashboard uses Intel sysfs GPU detection
      # regardless of .env state.
      - GPU_BACKEND=sycl
    volumes:
      - /sys/class/drm:/sys/class/drm:ro
      - /sys/class/hwmon:/sys/class/hwmon:ro