ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
# Host-managed Ollama / LM Studio overlay.
#
# The selected endpoint is reached through host.docker.internal. The explicit
# host-gateway mappings make that name available on plain Linux Docker Engine
# as well as Docker Desktop.
services:
litellm:
extra_hosts:
- "host.docker.internal:host-gateway"
volumes:
- type: bind
source: ./config/litellm/external-upstream.key
target: /run/secrets/ods-external-llm-api-key
read_only: true
bind:
create_host_path: false
selinux: z
llama-server:
profiles:
- local-inference
restart: "no"
model-router:
profiles:
- local-inference
restart: "no"
open-webui:
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
OPENAI_API_KEY: "${OPEN_WEBUI_LLM_API_KEY:-}"
dashboard-api:
extra_hosts:
- "host.docker.internal:host-gateway"
environment:
LLM_BACKEND: external
EXTERNAL_LLM_CONTAINER_URL: "${EXTERNAL_LLM_CONTAINER_URL}"
EXTERNAL_LLM_PROVIDER: "${EXTERNAL_LLM_PROVIDER:-openai-compatible}"
EXTERNAL_LLM_MODEL: "${EXTERNAL_LLM_MODEL:-}"
ODS_MODEL_SWITCHBOARD: "${ODS_MODEL_SWITCHBOARD:-observe}"
LLM_API_BASE_PATH: /v1
ODS_TALK_VISION_URL: "${EXTERNAL_LLM_CONTAINER_URL}/v1"
AMD_INFERENCE_RUNTIME: ""
AMD_INFERENCE_BACKEND: ""
AMD_INFERENCE_LOCATION: ""
AMD_INFERENCE_PORT: ""
AMD_INFERENCE_SUPPORTED_BACKENDS: ""
AMD_INFERENCE_RUNTIME_MODE: ""
AMD_INFERENCE_MANAGED: ""