Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

1.5 KB
Raw Blame
# Host-managed Ollama / LM Studio overlay.
#
# The selected endpoint is reached through host.docker.internal. The explicit
# host-gateway mappings make that name available on plain Linux Docker Engine
# as well as Docker Desktop.
services:
  litellm:
    extra_hosts:
      - "host.docker.internal:host-gateway"
    volumes:
      - type: bind
        source: ./config/litellm/external-upstream.key
        target: /run/secrets/ods-external-llm-api-key
        read_only: true
        bind:
          create_host_path: false
          selinux: z

  llama-server:
    profiles:
      - local-inference
    restart: "no"

  model-router:
    profiles:
      - local-inference
    restart: "no"

  open-webui:
    extra_hosts:
      - "host.docker.internal:host-gateway"
    environment:
      OPENAI_API_KEY: "${OPEN_WEBUI_LLM_API_KEY:-}"

  dashboard-api:
    extra_hosts:
      - "host.docker.internal:host-gateway"
    environment:
      LLM_BACKEND: external
      EXTERNAL_LLM_CONTAINER_URL: "${EXTERNAL_LLM_CONTAINER_URL}"
      EXTERNAL_LLM_PROVIDER: "${EXTERNAL_LLM_PROVIDER:-openai-compatible}"
      EXTERNAL_LLM_MODEL: "${EXTERNAL_LLM_MODEL:-}"
      ODS_MODEL_SWITCHBOARD: "${ODS_MODEL_SWITCHBOARD:-observe}"
      LLM_API_BASE_PATH: /v1
      ODS_TALK_VISION_URL: "${EXTERNAL_LLM_CONTAINER_URL}/v1"
      AMD_INFERENCE_RUNTIME: ""
      AMD_INFERENCE_BACKEND: ""
      AMD_INFERENCE_LOCATION: ""
      AMD_INFERENCE_PORT: ""
      AMD_INFERENCE_SUPPORTED_BACKENDS: ""
      AMD_INFERENCE_RUNTIME_MODE: ""
      AMD_INFERENCE_MANAGED: ""