ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
# ODS — Intel Arc GPU Overlay (oneAPI SYCL, build-from-source)
#
# Builds llama-server from source using Intel oneAPI Base Toolkit.
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.arc.yml up -d
#
# Supported hardware:
# ARC tier — Arc A770 16 GB (and future Arc B-series ≥12 GB)
# ARC_LITE tier — Arc A750 8 GB, A380 6 GB
#
# Build arguments (customise via .env or shell export):
# LLAMA_TAG llama.cpp git tag (default: b9014)
# LLAMA_COMMIT commit LLAMA_TAG must resolve to (default: b9014's commit);
# set it together with LLAMA_TAG, or empty to skip the check
# ONEAPI_VERSION oneAPI Base Toolkit image tag (default: 2025.0.0-0-devel-ubuntu22.04)
# LLAMA_ARC_IMAGE Override to a pre-built image and skip the local build entirely
# e.g. LLAMA_ARC_IMAGE=ghcr.io/ggml-org/llama.cpp:server-intel-b9014@sha256:9c7bbaad3663523a3deb8927d3cfbf58d33f00a7634c69843e9eeeda01568c1b
#
# First-run note:
# The SYCL build compiles llama.cpp with Intel icx/icpx. Allow 10–20 min
# on first `docker compose up --build`. Subsequent starts use Docker cache.
#
# Host prerequisites (Ubuntu/Debian):
# apt install intel-opencl-icd intel-level-zero-gpu level-zero
# usermod -aG video,render $USER # re-login after
# # Verify GPU: clinfo | grep -i "intel arc"
services:
llama-server:
build:
context: ./images/llama-sycl
dockerfile: Dockerfile
args:
LLAMA_TAG: ${LLAMA_TAG:-b9014}
LLAMA_COMMIT: ${LLAMA_COMMIT-d4b0c22f9e67f0295e91dc1ab4f17c0fb2557fa4}
ONEAPI_VERSION: ${ONEAPI_VERSION:-2025.0.0-0-devel-ubuntu22.04}
# Set LLAMA_ARC_IMAGE in .env to skip the local build and pull a pre-built image.
image: ${LLAMA_ARC_IMAGE:-ods-llama-sycl:local}
devices:
- /dev/dri:/dev/dri
group_add:
- "${VIDEO_GID:-44}"
- "${RENDER_GID:-992}"
environment:
# Level Zero selects the Intel Arc GPU.
# SYCL_CACHE_PERSISTENT is deliberately not set: with the oneAPI 2025.3
# runtime in the b9014 image it crashes llama-server (ggml-org/llama.cpp
# #21474, #22095; intel/llvm#22853). ONEAPI_DEVICE_SELECTOR can be set in
# .env; use level_zero:0 on hosts with more than one Intel GPU
# (ggml-org/llama.cpp#21747).
- ONEAPI_DEVICE_SELECTOR=${ONEAPI_DEVICE_SELECTOR:-level_zero:gpu}
# Enable Intel GPU System Management Interface for telemetry
- ZES_ENABLE_SYSMAN=1
command:
- --model
- /models/${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
# Same served id as docker-compose.base.yml: the GGUF file name.
- --alias
- ${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
- --host
- 0.0.0.0
- --port
- "8080"
- --n-gpu-layers
- "${N_GPU_LAYERS:-99}"
- --ctx-size
- "${CTX_SIZE:-32768}"
# Compose replaces `command` wholesale rather than merging it, so the
# documented runtime tunables have to be repeated here or they go inert
# on this overlay.
- --batch-size
- "${LLAMA_BATCH_SIZE:-2048}"
- --threads
- "${LLAMA_THREADS:-4}"
- --parallel
- "${LLAMA_PARALLEL:-1}"
- --metrics
deploy:
resources:
limits:
cpus: '${LLAMA_CPU_LIMIT:-16.0}'
memory: ${LLAMA_SERVER_MEMORY_LIMIT:-24G}
reservations:
cpus: '${LLAMA_CPU_RESERVATION:-2.0}'
memory: 4G
dashboard-api:
environment:
# Hard-code sycl so the dashboard uses Intel sysfs GPU detection
# regardless of .env state.
- GPU_BACKEND=sycl
volumes:
- /sys/class/drm:/sys/class/drm:ro
- /sys/class/hwmon:/sys/class/hwmon:ro