ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
# ODS — Core Service Definitions
# Extension services live in extensions/services/*/compose.yaml
# GPU overlays layered on top:
# docker compose -f docker-compose.base.yml -f docker-compose.amd.yml up -d
# docker compose -f docker-compose.base.yml -f docker-compose.nvidia.yml up -d
name: ods
# ============================================
# Standardized Logging Policy
# ============================================
x-logging: &default-logging
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
services:
# ============================================
# LLM Inference — llama-server (GPU-agnostic stub)
# Image and GPU config provided by overlay
# Serves OpenAI-compatible API on port 8080
# ============================================
llama-server:
image: ${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-b9014@sha256:2e7953dfef88f302bf0683bffa7dc1f8d86ef75910380bc41126ec5b8bedaf53}
container_name: ods-llama-server
restart: unless-stopped
# Normalize reasoning modes on every launch, including CLI restarts,
# model switches and direct Compose use. The wrapper execs the server.
entrypoint: ["/bin/sh", "/opt/ods/llama-server-entrypoint.sh"]
# config/llama-server/models.ini is not mounted: llama-server reads a
# preset file only with --models-preset (router mode), which ODS does not
# use; the model comes from --model below. ODS still writes models.ini as
# a record of the active model.
volumes:
- ./data/models:/models:z
# Fixed chat templates ODS vendors from llama.cpp (any-model WP5), read
# only when LLAMA_ARG_CHAT_TEMPLATE_FILE below names one.
- ./config/chat-templates:/chat-templates:ro,z
- ./extensions/services/llama-server/llama-server-entrypoint.sh:/opt/ods/llama-server-entrypoint.sh:ro,z
ports:
- "127.0.0.1:${OLLAMA_PORT:-11434}:8080"
command:
- --model
- /models/${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
# The served model id is the GGUF file name on every runtime, so
# /v1/models, model-state and every consumer use one identity.
- --alias
- ${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
- --host
- 0.0.0.0
- --port
- "8080"
- --n-gpu-layers
- "${N_GPU_LAYERS:-auto}"
- --ctx-size
- "${CTX_SIZE:-16384}"
- --batch-size
- "${LLAMA_BATCH_SIZE:-2048}"
- --threads
- "${LLAMA_THREADS:-4}"
- --parallel
- "${LLAMA_PARALLEL:-1}"
- --metrics
environment:
- LLAMA_ARG_REASONING=${LLAMA_REASONING:-off}
- LLAMA_ARG_FLASH_ATTN=${LLAMA_ARG_FLASH_ATTN:-auto}
- LLAMA_ARG_CACHE_TYPE_K=${LLAMA_ARG_CACHE_TYPE_K:-f16}
- LLAMA_ARG_CACHE_TYPE_V=${LLAMA_ARG_CACHE_TYPE_V:-f16}
# Optional MoE expert CPU offload. Do not default to an empty value:
# llama.cpp parses this as an integer and exits if it is blank.
- LLAMA_ARG_N_CPU_MOE
# Optional long-context runtime flags. Leave unset unless a model
# profile explicitly requires them. llama.cpp reads
# LLAMA_ARG_NO_CACHE_PROMPT through its negated-flag compatibility path:
# any value, even 0 or empty, disables prompt caching.
- LLAMA_ARG_NO_CACHE_PROMPT
- LLAMA_ARG_CHECKPOINT_EVERY_NT
# --checkpoint-min-step replaced --checkpoint-every-n-tokens in
# llama.cpp b9310 and has different semantics (minimum spacing, not a
# fixed interval). The pinned b9014 image reads only
# LLAMA_ARG_CHECKPOINT_EVERY_NT; b9310 and later read only this one.
- LLAMA_ARG_CHECKPOINT_MIN_SPACING_NT
# Host-RAM bounds for hybrid/sliding-window models: context checkpoints
# kept per slot (b9014 default 32) and the prompt cache in MiB (default
# 8192). CPU runtime profiles cap both; unset keeps llama.cpp defaults.
- LLAMA_ARG_CTX_CHECKPOINTS
- LLAMA_ARG_CACHE_RAM
# Vision projector of an imported model (/models/<mmproj file>). Set by
# model activation for models that bring one and removed otherwise;
# passed through only when set, so text models load without it.
- LLAMA_ARG_MMPROJ
# A fixed chat template (/chat-templates/upstream-b<build>/<name>.jinja)
# for a model whose own template is known to be broken. Set by model
# activation only on an exact template-hash match and removed for every
# other model; passed through only when set (an empty value would make
# llama.cpp fail to open a file named "").
- LLAMA_ARG_CHAT_TEMPLATE_FILE
# Per-model speculative decoding (e.g. draft-mtp from a runtime
# profile). Passed through only when set; the NVIDIA and CPU overlays
# default it to ${LLAMA_SPEC_TYPE:-ngram-mod}. The AMD, Intel, Arc and
# Apple overlays add no default.
- LLAMA_ARG_SPEC_TYPE
- LLAMA_ARG_SPEC_DRAFT_N_MAX
# Draft-model KV cache types (--spec-draft-type-k/-v). These are the
# names llama.cpp reads; ODS's native launchers take
# LLAMA_ARG_SPEC_DRAFT_TYPE_K/V instead and pass them as flags.
- LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_K
- LLAMA_ARG_SPEC_DRAFT_CACHE_TYPE_V
security_opt:
- no-new-privileges:true
logging: *default-logging
healthcheck:
test: ["CMD", "curl", "-sf", "http://127.0.0.1:8080/health"]
interval: 15s
timeout: 10s
retries: 5
# 70B-class GGUFs cold-load in 60-120s; thermal throttling and first-touch
# page faults push Tier 4 past 180s. Override via LLAMA_START_PERIOD in .env.
start_period: ${LLAMA_START_PERIOD:-240s}
# ============================================
# Chat UI
# ============================================
open-webui:
image: ghcr.io/open-webui/open-webui:v0.11.4@sha256:9591b13f13843c7721c2b8eaf7382846c81b3ffe126526d1888d1fed50c6a33f
container_name: ods-webui
restart: unless-stopped
logging: *default-logging
security_opt:
- no-new-privileges:true
# openwebui-prepare.py copies the database aside before a new Open WebUI
# version migrates it, and refuses a start that would leave it
# half-migrated. The wrapper then runs the image's own start command, which
# must be restated because Docker drops the image CMD when `entrypoint:` is
# set. The image's working directory (/app/backend) and user are kept.
entrypoint: ["/bin/sh", "/opt/ods/openwebui-entrypoint.sh"]
command: ["bash", "start.sh"]
environment:
# ODS owns Open WebUI's settings: they are read from this environment at
# every start, so mode switches, Pixel, key rotation and the Dashboard
# Settings page keep applying. Without this, Open WebUI 0.10 and later
# writes every setting into webui.db on its first start and ignores later
# changes. Changes made in Open WebUI's admin panel last until it restarts.
ENABLE_PERSISTENT_CONFIG: "false"
# Nothing stores "signup closed" any more, so it is set here. Open WebUI
# still lets the first account sign up and makes it the administrator;
# administrators add the other accounts in Admin Panel > Users.
ENABLE_SIGNUP: "false"
ENABLE_OLLAMA_API: "false"
# ODS pins and updates Open WebUI itself; the upstream "new version"
# check only calls GitHub and advertises releases ODS has not qualified.
ENABLE_VERSION_UPDATE_CHECK: "false"
# Arbitrary local models may answer chat but fail native tool calling.
# Explicit per-model or chat Native settings override this default.
DEFAULT_MODEL_PARAMS: '{"function_calling":"legacy"}'
# Switchboard: OPEN_WEBUI_LLM_BASE_URL overrides to the LiteLLM gateway
# (ods/current) in enabled mode; default is byte-identical to the prior
# direct llama-server route for legacy/observe.
OPENAI_API_BASE_URL: "${OPEN_WEBUI_LLM_BASE_URL:-${LLM_API_URL:-http://llama-server:8080}/v1}"
OPENAI_API_KEY: "${OPEN_WEBUI_LLM_API_KEY:-}"
# openwebui-model-list.py applies the LiteLLM config choice to the model
# picker; same defaults as the LiteLLM service.
ODS_MODE: "${ODS_MODE:-local}"
ODS_MODEL_SWITCHBOARD: "${ODS_MODEL_SWITCHBOARD:-enabled}"
# PWA / branding — surfaces as the app title, page title, and PWA install name.
# When a user adds ODS to their home screen, this is the label they see
# next to the ODS icon. Defaults to "ODS"; users can override per-install.
WEBUI_NAME: "${WEBUI_NAME:-ODS}"
# Optional public URL Open WebUI uses for share links, OAuth callbacks,
# and PWA install metadata. Leave empty for traditional localhost usage.
# Share links carry it, so set it only to a URL that actually resolves.
# Headless/proxy installs should set WEBUI_URL to the real reachable chat
# origin, usually http://chat.${ODS_DEVICE_NAME:-ods}.local once
# ods-proxy and mDNS are enabled. Tailscale/Cloudflare/custom domains
# should set this to the tunnel URL instead.
WEBUI_URL: "${WEBUI_URL:-}"
WEBUI_AUTH: "${WEBUI_AUTH:-true}"
# The address Open WebUI is published on (`ports:` below). The start-up
# wrapper turns sign-in on whenever it is not loopback, whatever .env
# says, so every way of recreating this container gets the same rule.
ODS_WEBUI_BIND_ADDRESS: "${BIND_ADDRESS:-127.0.0.1}"
WEBUI_SECRET_KEY: "${WEBUI_SECRET:?WEBUI_SECRET must be set — run the installer or add it to .env}"
ENABLE_WEB_SEARCH: "${ENABLE_WEB_SEARCH:-false}"
WEB_SEARCH_ENGINE: "searxng"
SEARXNG_QUERY_URL: "http://searxng:8080/search?q=<query>&format=json"
ENABLE_SEARCH_QUERY_GENERATION: "${ENABLE_WEB_SEARCH:-false}"
WEB_SEARCH_RESULT_COUNT: "5"
# Route Open WebUI retrieval embeddings through ODS's TEI service.
# Otherwise Open WebUI downloads sentence-transformers/all-MiniLM-L6-v2
# during app startup, which can block fresh installs before port 8080 binds.
RAG_EMBEDDING_ENGINE: "${RAG_EMBEDDING_ENGINE:-openai}"
# Use the exact model served by bundled TEI. Open WebUI reads it at every
# start (ENABLE_PERSISTENT_CONFIG above).
RAG_EMBEDDING_MODEL: "${RAG_EMBEDDING_MODEL:-${EMBEDDING_MODEL:-BAAI/bge-base-en-v1.5}}"
RAG_OPENAI_API_BASE_URL: "${RAG_OPENAI_API_BASE_URL:-http://embeddings:80/v1}"
RAG_OPENAI_API_KEY: "${RAG_OPENAI_API_KEY:-}"
RAG_EMBEDDING_MODEL_AUTO_UPDATE: "${RAG_EMBEDDING_MODEL_AUTO_UPDATE:-false}"
# ── ComfyUI Image Generation (SDXL Lightning 4-step) ──
# Default OFF in base: ComfyUI ships only on GPU backends (amd/nvidia per
# comfyui/manifest.yaml). The amd and nvidia overlays re-assert this as
# ${...:-true} so image-gen is on for GPU users by default. CPU/Apple
# users would otherwise hit a missing-engine error from Open WebUI.
ENABLE_IMAGE_GENERATION: "${ENABLE_IMAGE_GENERATION:-false}"
IMAGE_GENERATION_ENGINE: "comfyui"
# Owner-settable in .env (Settings applies them by recreating Open WebUI).
# The workflow below stays fixed; IMAGE_GENERATION_MODEL fills its loader.
COMFYUI_BASE_URL: "${COMFYUI_BASE_URL:-http://comfyui:8188}"
IMAGE_SIZE: "${IMAGE_SIZE:-1024x1024}"
IMAGE_STEPS: "${IMAGE_STEPS:-4}"
IMAGE_GENERATION_MODEL: "${IMAGE_GENERATION_MODEL:-sdxl_lightning_4step.safetensors}"
COMFYUI_WORKFLOW: >-
{"3":{"inputs":{"seed":42,"steps":4,"cfg":1.5,"sampler_name":"euler","scheduler":"sgm_uniform","denoise":1.0,"model":["4",0],"positive":["6",0],"negative":["7",0],"latent_image":["5",0]},"class_type":"KSampler"},
"4":{"inputs":{"ckpt_name":"sdxl_lightning_4step.safetensors"},"class_type":"CheckpointLoaderSimple"},
"5":{"inputs":{"width":1024,"height":1024,"batch_size":1},"class_type":"EmptyLatentImage"},
"6":{"inputs":{"text":"placeholder","clip":["4",1]},"class_type":"CLIPTextEncode"},
"7":{"inputs":{"text":"","clip":["4",1]},"class_type":"CLIPTextEncode"},
"8":{"inputs":{"samples":["3",0],"vae":["4",2]},"class_type":"VAEDecode"},
"9":{"inputs":{"filename_prefix":"openwebui","images":["8",0]},"class_type":"SaveImage"}}
COMFYUI_WORKFLOW_NODES: >-
[{"type":"prompt","key":"text","node_ids":["6"]},
{"type":"model","key":"ckpt_name","node_ids":["4"]},
{"type":"width","key":"width","node_ids":["5"]},
{"type":"height","key":"height","node_ids":["5"]},
{"type":"steps","key":"steps","node_ids":["3"]},
{"type":"seed","key":"seed","node_ids":["3"]},
{"type":"n","key":"batch_size","node_ids":["5"]}]
# ── Speech-to-Text (Whisper) ──
AUDIO_STT_ENGINE: "${AUDIO_STT_ENGINE:-openai}"
AUDIO_STT_OPENAI_API_BASE_URL: "${AUDIO_STT_OPENAI_API_BASE_URL:-http://whisper:8000/v1}"
AUDIO_STT_OPENAI_API_KEY: "${AUDIO_STT_OPENAI_API_KEY:-}"
AUDIO_STT_MODEL: "${AUDIO_STT_MODEL:-Systran/faster-whisper-base}"
# ── Text-to-Speech (Kokoro) ──
AUDIO_TTS_ENGINE: "${AUDIO_TTS_ENGINE:-openai}"
AUDIO_TTS_OPENAI_API_BASE_URL: "${AUDIO_TTS_OPENAI_API_BASE_URL:-http://tts:8880/v1}"
AUDIO_TTS_OPENAI_API_KEY: "${AUDIO_TTS_OPENAI_API_KEY:-}"
AUDIO_TTS_MODEL: "${AUDIO_TTS_MODEL:-kokoro}"
AUDIO_TTS_VOICE: "${AUDIO_TTS_VOICE:-af_heart}"
TZ: "${TIMEZONE:-UTC}"
volumes:
- ./data/open-webui:/app/backend/data:z
- ./extensions/services/open-webui/openwebui-entrypoint.sh:/opt/ods/openwebui-entrypoint.sh:ro,z
- ./extensions/services/open-webui/openwebui-prepare.py:/opt/ods/openwebui-prepare.py:ro,z
- ./extensions/services/open-webui/openwebui-model-list.py:/opt/ods/openwebui-model-list.py:ro,z
ports:
- "${BIND_ADDRESS:-127.0.0.1}:${WEBUI_PORT:-3000}:8080"
deploy:
resources:
limits:
cpus: '2.0'
memory: 4G
reservations:
cpus: '0.25'
memory: 512M
healthcheck:
test: ["CMD", "curl", "-f", "http://127.0.0.1:8080/health"]
interval: 30s
timeout: 10s
retries: 3
start_period: 60s
# ============================================
# Dashboard API (System Status Backend)
# ============================================
dashboard-api:
build:
context: ./extensions/services/dashboard-api
dockerfile: Dockerfile
container_name: ods-dashboard-api
restart: unless-stopped
extra_hosts:
- "host.docker.internal:host-gateway"
# remote-provider is internal, so ods-network stays this container's
# default route; config.py finds the host agent through that gateway.
networks:
- default
- remote-provider
ports:
- "127.0.0.1:${DASHBOARD_API_PORT:-3002}:3002"
security_opt:
- no-new-privileges:true
logging: *default-logging
environment:
- ODS_INSTALL_DIR=/ods
- ODS_DATA_DIR=/data
- ODS_EXTENSIONS_LIBRARY_DIR=/data/extensions-library
- ODS_MODE=${ODS_MODE:-local}
# Service status reads this live from .env. An install owner other than
# UID 1000 leaves the 0600 .env unreadable here, and the status must then
# fall back to this copy instead of counting Open WebUI as a core service down.
- ENABLE_OPEN_WEBUI=${ENABLE_OPEN_WEBUI:-true}
# The same fallback keeps --offline's switch off the GitHub release check.
- DISABLE_UPDATE_CHECK=${DISABLE_UPDATE_CHECK:-false}
- ODS_AGENT_HOST=${ODS_AGENT_HOST:-}
- ODS_AGENT_PORT=${ODS_AGENT_PORT:-7710}
- GPU_BACKEND=${GPU_BACKEND:-nvidia}
- GPU_COUNT=${GPU_COUNT:-1}
- GPU_ASSIGNMENT_JSON_B64=${GPU_ASSIGNMENT_JSON_B64:-}
# The GPU page leaves CPU Whisper off its assigned GPU; the same
# unreadable-.env fallback as ENABLE_OPEN_WEBUI above.
- WHISPER_ACCELERATION=${WHISPER_ACCELERATION:-}
- LLAMA_ARG_SPLIT_MODE=${LLAMA_ARG_SPLIT_MODE:-}
- LLAMA_ARG_TENSOR_SPLIT=${LLAMA_ARG_TENSOR_SPLIT:-}
- HOST_RAM_GB=${HOST_RAM_GB:-0}
# WSL may use the CPU compose surface while inference runs on the
# Windows GPU. Pass the selected runtime independently of GPU exposure.
- LLM_BACKEND=${LLM_BACKEND:-}
- AMD_INFERENCE_LOCATION=${AMD_INFERENCE_LOCATION:-}
- AMD_INFERENCE_PORT=${AMD_INFERENCE_PORT:-8080}
# Origin of an ODS-managed llama-server outside this stack (Windows
# Portal), as seen from containers and from the host. Empty otherwise.
- NATIVE_LLM_CONTAINER_BASE_URL=${NATIVE_LLM_CONTAINER_BASE_URL:-}
- NATIVE_LLM_BASE_URL=${NATIVE_LLM_BASE_URL:-}
- OLLAMA_URL=${LLM_API_URL:-http://llama-server:8080}
- LLM_MODEL=${LLM_MODEL:-qwen3:30b-a3b}
- GGUF_FILE=${GGUF_FILE:-}
- KOKORO_URL=${KOKORO_URL:-http://tts:8880}
- N8N_URL=${N8N_URL:-http://n8n:5678}
- DASHBOARD_API_KEY=${DASHBOARD_API_KEY:-}
- ODS_AGENT_KEY=${ODS_AGENT_KEY:-}
- ODS_ROUTER_INTERNAL_KEY=${ODS_ROUTER_INTERNAL_KEY:-}
- MODEL_ROUTER_URL=http://model-router:9099
- SHIELD_API_KEY=${SHIELD_API_KEY:-}
- TOKEN_SPY_URL=${TOKEN_SPY_URL:-http://token-spy:8080}
- TOKEN_SPY_API_KEY=${TOKEN_SPY_API_KEY:-}
# Signing secret for ods-session cookies. session_signer.py reads
# this at import time; if empty, issue() raises and verify() returns
# (False, "no-secret"). The installer generates a random value at
# phase 06; an unconfigured dev box without the secret will fail
# magic-link redemption loudly rather than mint unsignable cookies.
- ODS_SESSION_SECRET=${ODS_SESSION_SECRET:-}
# Device/public URL inputs used by magic-link generation. Keep these
# aligned with ods-proxy and ods-mdns so invite QR codes point at
# the actual LAN-facing auth/chat hosts for the unit.
- ODS_DEVICE_NAME=${ODS_DEVICE_NAME:-ods}
- ODS_PUBLIC_URL=${ODS_PUBLIC_URL:-}
# Cookie Domain scope. Set to <device>.local (no leading dot) so
# cookies are shared across chat / auth / hermes / dashboard
# subdomains routed by the ods-proxy. Empty = host-only (only
# the redemption host sees the cookie); functional for single-host
# setups but breaks subdomain SSO.
- ODS_COOKIE_DOMAIN=${ODS_COOKIE_DOMAIN:-}
# URLs consumed by scripts/ods-test-functional.sh (executed inside this
# container by /api/setup/test). The script's own `localhost:<port>`
# defaults are correct for host-side runs but unreachable from inside
# the container; explicitly point at docker network names here.
- LLM_URL=${LLM_URL:-${LLM_API_URL:-http://llama-server:8080}}
- TTS_URL=${TTS_URL:-http://tts:8880}
- EMBEDDING_URL=${EMBEDDING_URL:-http://embeddings:80}
- WHISPER_URL=${WHISPER_URL:-http://whisper:8000}
# ODS Talk's image-attachment endpoint calls the vision backend
# directly (bypassing litellm because litellm's wildcard model
# routing collapses our model id down to whatever's loaded).
# Defaults point at the llama-server OpenAI-compatible base URL on the
# internal docker network — no auth needed there. Operators with an
# external vision server set ODS_TALK_VISION_URL +
# ODS_TALK_VISION_KEY.
- ODS_TALK_VISION_URL=${ODS_TALK_VISION_URL:-http://llama-server:8080${LLM_API_BASE_PATH:-/v1}}
- ODS_TALK_VISION_KEY=${ODS_TALK_VISION_KEY:-}
# Local models can take several minutes to produce the first useful
# ODS Talk frame on cold sessions. Keep the dashboard-api bridge aligned
# with Hermes/LiteLLM long-model timeouts so Talk does not 502 while the
# underlying agent is still working.
- ODS_TALK_HERMES_TIMEOUT=${ODS_TALK_HERMES_TIMEOUT:-900}
# Shared managed login defaults; raw secrets never leave the server.
- HERMES_DASHBOARD_SESSION_TOKEN=${HERMES_DASHBOARD_SESSION_TOKEN:-}
- HERMES_ODS_MANAGED_AUTH=${HERMES_ODS_MANAGED_AUTH:-true}
- HERMES_DASHBOARD_BASIC_AUTH_USERNAME=${HERMES_DASHBOARD_BASIC_AUTH_USERNAME:-}
- HERMES_DASHBOARD_BASIC_AUTH_PASSWORD=${HERMES_DASHBOARD_BASIC_AUTH_PASSWORD:-}
- HERMES_DASHBOARD_BASIC_AUTH_PASSWORD_HASH=${HERMES_DASHBOARD_BASIC_AUTH_PASSWORD_HASH:-}
- HERMES_DASHBOARD_OAUTH_CLIENT_ID=${HERMES_DASHBOARD_OAUTH_CLIENT_ID:-}
- HERMES_DASHBOARD_OIDC_CLIENT_ID=${HERMES_DASHBOARD_OIDC_CLIENT_ID:-}
# Model id for image attachments. Empty means the active model: there is
# no separately registered vision model on any runtime. Set it only
# together with ODS_TALK_VISION_URL for an external vision server.
- ODS_TALK_VISION_MODEL=${ODS_TALK_VISION_MODEL:-}
volumes:
# .env is atomically replaced; mounting the parent keeps updates visible.
- ./:/ods:ro,z
- ./data:/data:z
deploy:
resources:
limits:
cpus: '1.0'
memory: 2G
reservations:
cpus: '0.25'
memory: 512M
healthcheck:
test: ["CMD", "curl", "-f", "http://127.0.0.1:3002/health"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
# ============================================
# Model Router (Switchboard stable-alias data plane)
# ============================================
model-router:
build:
context: ./extensions/services/model-router
dockerfile: Dockerfile
container_name: ods-model-router
restart: unless-stopped
expose:
- "9099"
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
read_only: true
logging: *default-logging
environment:
- ODS_MODEL_STATE_PATH=/state/model-state.json
- ODS_ROUTER_ENDPOINTS_PATH=/config/endpoints.json
- ODS_ROUTER_INTERNAL_KEY=${ODS_ROUTER_INTERNAL_KEY:-}
- DASHBOARD_API_KEY=${DASHBOARD_API_KEY:-}
- ODS_FLEET_PROBE_KEY=${ODS_FLEET_PROBE_KEY:-}
- TOKEN_SPY_URL=${TOKEN_SPY_URL:-http://token-spy:8080}
- TOKEN_SPY_API_KEY=${TOKEN_SPY_API_KEY:-}
# File-based probe key inside the already-mounted state volume: the
# fleet harness can create/rotate/remove it without recreating the
# router (a recreate destroys in-memory route evidence). Absent file
# means the ODS_FLEET_PROBE_KEY env fallback applies unchanged.
- ODS_FLEET_PROBE_KEY_PATH=/state/fleet-probe.key
volumes:
- ${ODS_DATA_DIR:-./data}:/state:ro
# The renderer atomically replaces endpoints.json; mount its parent so
# the router sees the new inode without a container recreate.
- ${ODS_CONFIG_DIR:-./config}/model-router:/config:ro
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
reservations:
cpus: '0.1'
memory: 64M
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:9099/health',timeout=4).status==200 else 1)"]
interval: 15s
timeout: 5s
retries: 3
start_period: 10s
# ============================================
# Remote Provider Egress (remote LLM boundary)
# ============================================
remote-provider-egress:
build:
context: .
dockerfile: extensions/services/remote-provider-egress/Dockerfile
image: ods-remote-provider-egress:local
container_name: ods-remote-provider-egress
restart: unless-stopped
expose:
- "8091"
# Not on ods-network: only LiteLLM and dashboard-api share the internal
# remote-provider network, and the outbound network carries provider
# traffic. Extensions on ods-network cannot reach this service.
networks:
- remote-provider
- remote-provider-outbound
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
# Lifecycle secrets stay owner-written on the host and group-readable only
# to the two dedicated, read-only remote-provider containers.
group_add:
- "${REMOTE_PROVIDER_DATA_GID:-1000}"
read_only: true
tmpfs:
- /tmp:size=16m,mode=1777
logging: *default-logging
environment:
- ODS_REMOTE_PROVIDER_ROUTE_PATH=/state/remote-provider/routing-state.json
- ODS_REMOTE_PROVIDER_POLICY_PATH=/config/remote-provider-egress-policy.json
- ODS_REMOTE_PROVIDER_API_KEY_FILE=/state/remote-provider/secrets/provider-api-key
# Callers must present the LiteLLM gateway key (LiteLLM, dashboard-api).
- ODS_REMOTE_PROVIDER_CALLER_KEY=${LITELLM_KEY:-}
- ODS_REMOTE_PROVIDER_MAX_BODY_BYTES=${ODS_REMOTE_PROVIDER_MAX_BODY_BYTES:-4194304}
- ODS_REMOTE_PROVIDER_UPSTREAM_TIMEOUT=${ODS_REMOTE_PROVIDER_UPSTREAM_TIMEOUT:-600}
- ODS_REMOTE_PROVIDER_SSH_TUNNEL_HEALTH_URL=${ODS_REMOTE_PROVIDER_SSH_TUNNEL_HEALTH_URL:-http://remote-provider-ssh-tunnel:18090/health}
- ODS_REMOTE_PROVIDER_SSH_TUNNEL_HEALTH_TIMEOUT=${ODS_REMOTE_PROVIDER_SSH_TUNNEL_HEALTH_TIMEOUT:-2}
volumes:
- ${ODS_DATA_DIR:-./data}/remote-provider:/state/remote-provider:ro
- ${ODS_CONFIG_DIR:-./config}/remote-provider-egress-policy.json:/config/remote-provider-egress-policy.json:ro
deploy:
resources:
limits:
cpus: '0.5'
memory: 256M
reservations:
cpus: '0.1'
memory: 64M
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8091/health',timeout=4).status==200 else 1)"]
interval: 15s
timeout: 5s
retries: 3
start_period: 10s
# ============================================
# Remote Provider SSH Tunnel (remote LLM SSH transport)
# ============================================
remote-provider-ssh-tunnel:
build:
context: .
dockerfile: extensions/services/remote-provider-ssh-tunnel/Dockerfile
image: ods-remote-provider-ssh-tunnel:local
container_name: ods-remote-provider-ssh-tunnel
restart: unless-stopped
expose:
- "18090"
- "18091"
- "18092"
# Its SSH forwards are plain TCP with no caller check, so they must not be
# reachable from ods-network; see remote-provider-egress.
networks:
- remote-provider
- remote-provider-outbound
security_opt:
- no-new-privileges:true
cap_drop:
- ALL
group_add:
- "${REMOTE_PROVIDER_DATA_GID:-1000}"
read_only: true
tmpfs:
- /tmp:size=16m,mode=1777
logging: *default-logging
environment:
- ODS_REMOTE_PROVIDER_ROUTE_PATH=/state/remote-provider/routing-state.json
- ODS_REMOTE_PROVIDER_SECRET_DIR=/state/remote-provider/secrets
- ODS_REMOTE_PROVIDER_SSH_STATUS_PORT=18090
- ODS_REMOTE_PROVIDER_SSH_RECONCILE_INTERVAL=${ODS_REMOTE_PROVIDER_SSH_RECONCILE_INTERVAL:-5}
- ODS_REMOTE_PROVIDER_SSH_START_GRACE=${ODS_REMOTE_PROVIDER_SSH_START_GRACE:-1}
- ODS_REMOTE_PROVIDER_SSH_RESTART_COOLDOWN=${ODS_REMOTE_PROVIDER_SSH_RESTART_COOLDOWN:-5}
- ODS_REMOTE_PROVIDER_SSH_STOP_TIMEOUT=${ODS_REMOTE_PROVIDER_SSH_STOP_TIMEOUT:-5}
volumes:
- ${ODS_DATA_DIR:-./data}/remote-provider:/state/remote-provider:ro
deploy:
resources:
limits:
cpus: '0.5'
memory: 128M
reservations:
cpus: '0.05'
memory: 32M
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request,sys;sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:18090/health',timeout=4).status==200 else 1)"]
interval: 15s
timeout: 5s
retries: 3
start_period: 10s
# ============================================
# Dashboard UI (Control Center)
# ============================================
dashboard:
build:
context: ./extensions/services/dashboard
dockerfile: Dockerfile
container_name: ods-dashboard
restart: unless-stopped
security_opt:
- no-new-privileges:true
logging: *default-logging
environment:
- DASHBOARD_API_KEY=${DASHBOARD_API_KEY:-}
- PIXEL_PREVIEW_PORT=${PIXEL_PREVIEW_PORT:-9437}
# Port 3001 is always host-loopback, even when other ODS services are
# network-bound. The separate port 3011 always requires sign-in.
- ODS_DASHBOARD_BIND=127.0.0.1
volumes:
- ./data:/data:ro,z
ports:
- "127.0.0.1:${DASHBOARD_PORT:-3001}:3001"
- "${BIND_ADDRESS:-127.0.0.1}:${DASHBOARD_REMOTE_PORT:-3011}:3011"
deploy:
resources:
limits:
cpus: '1.0'
memory: 2G
reservations:
cpus: '0.25'
memory: 512M
depends_on:
dashboard-api:
condition: service_healthy
healthcheck:
test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://127.0.0.1:3001/"]
interval: 30s
timeout: 5s
retries: 3
start_period: 10s
networks:
default:
name: ods-network
# The remote-provider egress boundary. Internal, so it never becomes a
# member's default route; only LiteLLM, dashboard-api, the egress and the
# SSH tunnel join it, and extensions may not (shared compose policy).
remote-provider:
name: ods-remote-provider
internal: true
# Internet access for the egress and the SSH tunnel only.
remote-provider-outbound:
name: ods-remote-provider-outbound