Skip to content

armgit / ODS

Public Imported from GitHub
0

ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.

1.5 KB
Raw Blame
# ODS — CPU-Only Overlay (Linux / no GPU)
# Uses llama.cpp server CPU build. No NVIDIA/AMD/Intel GPU required.
# For low-RAM systems, set LLAMA_SERVER_MEMORY_LIMIT=4G in .env
#
# Use with: docker compose -f docker-compose.base.yml -f docker-compose.cpu.yml up -d

services:
  llama-server:
    # b8248 ignores LLAMA_ARG_REASONING from the base service; b9014 honors it.
    image: ${LLAMA_SERVER_IMAGE:-ghcr.io/ggml-org/llama.cpp:server-b9014@sha256:2e7953dfef88f302bf0683bffa7dc1f8d86ef75910380bc41126ec5b8bedaf53}
    command:
      - --model
      - /models/${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
      # Same served id as docker-compose.base.yml: the GGUF file name.
      - --alias
      - ${GGUF_FILE:-Qwen3.5-9B-Q4_K_M.gguf}
      - --host
      - 0.0.0.0
      - --port
      - "8080"
      - --n-gpu-layers
      - "0"
      - --ctx-size
      - "${CTX_SIZE:-8192}"
      - --batch-size
      - "${LLAMA_BATCH_SIZE:-512}"
      - --threads
      - "${LLAMA_THREADS:-4}"
      - --parallel
      - "${LLAMA_PARALLEL:-1}"
      - --metrics
    environment:
      # n-gram speculative decoding; same contract as docker-compose.nvidia.yml.
      # The pinned b9014 image has the b8955+ ngram-mod implementation. Set
      # LLAMA_SPEC_TYPE=none in .env to turn it off.
      - LLAMA_ARG_SPEC_TYPE=${LLAMA_ARG_SPEC_TYPE:-${LLAMA_SPEC_TYPE:-ngram-mod}}
    deploy:
      resources:
        limits:
          cpus: '${LLAMA_CPU_LIMIT:-8.0}'
          memory: ${LLAMA_SERVER_MEMORY_LIMIT:-6G}
        reservations:
          cpus: '${LLAMA_CPU_RESERVATION:-1.0}'
          memory: 2G