Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

7.1 KB
Raw Blame
# Debug layer - adds debugpy to services
#
# Usage:
#   VERSION=test docker compose -f docker-compose.yaml -f compose.override.yaml -f compose.debug.yaml watch
#
# Debug Port Map:
#   5678 - backend
#   5679 - runner
#   5680 - platform-service
#   5682 - worker-pg-fileproc
#   5683 - worker-pg-callback
#   5684 - worker-pg-orchestrator-api
#   5685 - worker-pg-orchestrator-general
#   5686 - worker-pg-notification
#   5687 - worker-log-stream-consumer
#   5688 - worker-pg-scheduler

services:
  #########################################################################################################
  # Python Services with debugpy

  backend:
    ports:
      - "5678:5678"
    command: [
      "uv run python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 .venv/bin/gunicorn
        --bind 0.0.0.0:8000
        --workers 1
        --threads 2
        --worker-class gthread
        --log-level debug
        --timeout 900
        --graceful-timeout 5 backend.wsgi:application"
    ]

  runner:
    ports:
      - "5679:5679"
    command: [
      "uv run python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5679 .venv/bin/gunicorn
        --bind 0.0.0.0:5002
        --workers 1
        --threads 2
        --worker-class gthread
        --log-level debug
        --timeout 900
        --access-logfile -
        --graceful-timeout 5 unstract.runner:app"
    ]

  platform-service:
    ports:
      - "3001:3001"
      - "5680:5680"
    command: [
      "uv run python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5680 .venv/bin/gunicorn
        --bind 0.0.0.0:3001
        --workers 1
        --threads 2
        --worker-class gthread
        --log-level debug
        --timeout 900
        --access-logfile -
        --graceful-timeout 5 unstract.platform_service.run:app"
    ]

  #########################################################################################################
  # V2 Workers with debugpy
  # Using --pool=threads for debugpy compatibility (prefork doesn't work well with debugpy)

  worker-pg-fileproc:
    user: root
    ports:
      - "5682:5682"
    entrypoint: ["bash", "-c"]
    # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
    # override REPLACES the base file's command, so a Celery invocation here
    # turns the service into a RabbitMQ worker while the backend enqueues to
    # Postgres — the debug stack then accepts work and runs none of it.
    #
    # Queue set and worker type are deliberately NOT repeated: the module reads
    # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
    # environment. Restating them here is what let this file drift out of sync.
    command: [
      "uv run python -m debugpy --listen 0.0.0.0:5682 -m pg_queue_consumer"
    ]

  worker-pg-callback:
    user: root
    ports:
      - "5683:5683"
    entrypoint: ["bash", "-c"]
    # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
    # override REPLACES the base file's command, so a Celery invocation here
    # turns the service into a RabbitMQ worker while the backend enqueues to
    # Postgres — the debug stack then accepts work and runs none of it.
    #
    # Queue set and worker type are deliberately NOT repeated: the module reads
    # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
    # environment. Restating them here is what let this file drift out of sync.
    command: [
      "uv run python -m debugpy --listen 0.0.0.0:5683 -m pg_queue_consumer"
    ]

  worker-pg-orchestrator-api:
    user: root
    ports:
      - "5684:5684"
    entrypoint: ["bash", "-c"]
    # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
    # override REPLACES the base file's command, so a Celery invocation here
    # turns the service into a RabbitMQ worker while the backend enqueues to
    # Postgres — the debug stack then accepts work and runs none of it.
    #
    # Queue set and worker type are deliberately NOT repeated: the module reads
    # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
    # environment. Restating them here is what let this file drift out of sync.
    command: [
      "uv run python -m debugpy --listen 0.0.0.0:5684 -m pg_queue_consumer"
    ]

  worker-pg-orchestrator-general:
    user: root
    ports:
      - "5685:5685"
    entrypoint: ["bash", "-c"]
    # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
    # override REPLACES the base file's command, so a Celery invocation here
    # turns the service into a RabbitMQ worker while the backend enqueues to
    # Postgres — the debug stack then accepts work and runs none of it.
    #
    # Queue set and worker type are deliberately NOT repeated: the module reads
    # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
    # environment. Restating them here is what let this file drift out of sync.
    command: [
      "uv run python -m debugpy --listen 0.0.0.0:5685 -m pg_queue_consumer"
    ]

  worker-pg-notification:
    user: root
    ports:
      - "5686:5686"
    entrypoint: ["bash", "-c"]
    # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
    # override REPLACES the base file's command, so a Celery invocation here
    # turns the service into a RabbitMQ worker while the backend enqueues to
    # Postgres — the debug stack then accepts work and runs none of it.
    #
    # Queue set and worker type are deliberately NOT repeated: the module reads
    # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
    # environment. Restating them here is what let this file drift out of sync.
    command: [
      "uv run python -m debugpy --listen 0.0.0.0:5686 -m pg_queue_consumer"
    ]

  worker-log-stream-consumer:
    user: root
    ports:
      - "5687:5687"
    entrypoint: ["bash", "-c"]
    # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
    # override REPLACES the base file's command, so a Celery invocation here
    # turns the service into a RabbitMQ worker while the backend enqueues to
    # Postgres — the debug stack then accepts work and runs none of it.
    #
    # Queue set and worker type are deliberately NOT repeated: the module reads
    # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
    # environment. Restating them here is what let this file drift out of sync.
    command: [
      "uv run python -m debugpy --listen 0.0.0.0:5687 -m log_consumer.redis_stream_consumer"
    ]

  worker-pg-scheduler:
    user: root
    ports:
      - "5688:5688"
    entrypoint: ["bash", "-c"]
    # Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
    # override REPLACES the base file's command, so a Celery invocation here
    # turns the service into a RabbitMQ worker while the backend enqueues to
    # Postgres — the debug stack then accepts work and runs none of it.
    #
    # Queue set and worker type are deliberately NOT repeated: the module reads
    # WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
    # environment. Restating them here is what let this file drift out of sync.
    command: [
      "uv run python -m debugpy --listen 0.0.0.0:5688 -m pg_queue_consumer"
    ]