Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

10.5 KB
Raw Blame
# Development override - NO debugpy (use compose.debug.yaml for debugging)
#
# Usage:
#   Normal dev:  VERSION=test docker compose -f docker-compose.yaml -f compose.override.yaml watch
#   With debug:  VERSION=test docker compose -f docker-compose.yaml -f compose.override.yaml -f compose.debug.yaml watch
#
# Memory-optimized settings: reduced workers/threads for local development
#
# Requirements:
#   - Docker Compose 2.24.4+ (for !override directive support)

services:
  #########################################################################################################
  # Frontend - React dev server with hot reload
  frontend:
    image: unstract/frontend:${VERSION}
    build:
      dockerfile: docker/dockerfiles/frontend.Dockerfile
      context: ..
      target: development
    depends_on: !override
      - reverse-proxy
    env_file:
      - ../frontend/.env
    develop:
      watch:
        - action: sync
          path: ../frontend/
          target: /app
          ignore:
            - node_modules/
            - build/
        - action: rebuild
          path: ../frontend/bun.lock
        - action: rebuild
          path: ../frontend/package.json
    labels:
      - traefik.http.services.frontend.loadbalancer.server.port=3000

  #########################################################################################################
  # Backend - Gunicorn with reload (memory optimized: 1 worker, 2 threads)
  backend:
    image: unstract/backend:${VERSION}
    build:
      dockerfile: docker/dockerfiles/backend.Dockerfile
      context: ..
    depends_on: !override
      - db
      - redis
      - reverse-proxy
      - minio
      - platform-service
    init: true
    entrypoint: ["bash", "-c"]
    command: [
      "uv run python -Xfrozen_modules=off .venv/bin/gunicorn
        --bind 0.0.0.0:8000
        --workers 1
        --threads 2
        --worker-class gthread
        --log-level debug
        --timeout 900
        --access-logfile -
        --graceful-timeout 5 backend.wsgi:application"
    ]
    develop:
      watch:
        - action: sync+restart
          path: ../backend/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
    labels:
      - traefik.http.services.backend.loadbalancer.server.port=8000

  #########################################################################################################
  # Runner (memory optimized: 1 worker, 2 threads)
  runner:
    image: unstract/runner:${VERSION}
    build:
      dockerfile: docker/dockerfiles/runner.Dockerfile
      context: ..
    entrypoint: ["bash", "-c"]
    command: [
      "uv run python -Xfrozen_modules=off .venv/bin/gunicorn
        --bind 0.0.0.0:5002
        --workers 1
        --threads 2
        --worker-class gthread
        --log-level debug
        --timeout 900
        --access-logfile -
        --graceful-timeout 5 unstract.runner:app"
    ]
    develop:
      watch:
        - action: sync+restart
          path: ../runner/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../runner/uv.lock

  #########################################################################################################
  # Platform Service (memory optimized: 1 worker, 2 threads)
  platform-service:
    image: unstract/platform-service:${VERSION}
    build:
      dockerfile: docker/dockerfiles/platform.Dockerfile
      context: ..
    ports:
      - "3001:3001"
    entrypoint: ["bash", "-c"]
    command: [
      "uv run python -Xfrozen_modules=off .venv/bin/gunicorn
        --bind 0.0.0.0:3001
        --workers 1
        --threads 2
        --worker-class gthread
        --log-level debug
        --timeout 900
        --access-logfile -
        --graceful-timeout 5 unstract.platform_service.run:app"
    ]
    develop:
      watch:
        - action: sync+restart
          path: ../platform-service/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../platform-service/uv.lock

  #########################################################################################################
  # X2Text Service
  x2text-service:
    image: unstract/x2text-service:${VERSION}
    profiles:
      - optional
    build:
      dockerfile: docker/dockerfiles/x2text.Dockerfile
      context: ..

  #########################################################################################################
  # Tool Sidecar
  tool-sidecar:
    image: unstract/tool-sidecar:${VERSION}
    profiles:
      - tools
    build:
      dockerfile: docker/dockerfiles/tool-sidecar.Dockerfile
      context: ..

  #########################################################################################################
  # V2 Workers - Build configurations with memory-optimized settings
  # All use --pool=threads and --concurrency=2 for lower memory footprint

  worker-pg-orchestrator-api:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../workers/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../workers/uv.lock

  worker-pg-callback:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../workers/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../workers/uv.lock

  worker-pg-fileproc:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../workers/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../workers/uv.lock

  worker-pg-orchestrator-general:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../workers/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../workers/uv.lock

  worker-pg-notification:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../workers/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../workers/uv.lock

  worker-log-stream-consumer:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../workers/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../workers/uv.lock

  # No develop.watch - runs bash scheduler script, not Python code
  worker-log-history-scheduler-v2:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..

  worker-pg-scheduler:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../workers/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: rebuild
          path: ../workers/uv.lock

  #########################################################################################################
  # IDE callback worker (workers image, processes ide_callback queue)
  worker-pg-ide-callback:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
  # PG consumer for the dashboard_metrics.* periodics fired by the PG scheduler
  # tick. Must build the WORKER image: the base file runs `pg-queue-consumer`,
  # which the backend image's entrypoint rejects outright ("Unknown argument"),
  # so building backend.Dockerfile here crash-loops the pod and leaves
  # dashboard_metric_events with no consumer. No `command:` override either —
  # the base file's is correct.
  worker-pg-metrics:
    build:
      dockerfile: docker/dockerfiles/worker-unified.Dockerfile
      context: ..
    develop:
      watch:
        - action: sync+restart
          path: ../backend/
          target: /app
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
        - action: sync+restart
          path: ../unstract/
          target: /unstract
          ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]

  # V1 workers disabled by default (use workers-v2 profile instead)
  worker:
    profiles:
      - workers-v1

  worker-file-processing:
    profiles:
      - workers-v1

  worker-file-processing-callback:
    profiles:
      - workers-v1

  worker-logging:
    profiles:
      - workers-v1