LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
# Debug layer - adds debugpy to services
#
# Usage:
# VERSION=test docker compose -f docker-compose.yaml -f compose.override.yaml -f compose.debug.yaml watch
#
# Debug Port Map:
# 5678 - backend
# 5679 - runner
# 5680 - platform-service
# 5682 - worker-pg-fileproc
# 5683 - worker-pg-callback
# 5684 - worker-pg-orchestrator-api
# 5685 - worker-pg-orchestrator-general
# 5686 - worker-pg-notification
# 5687 - worker-log-stream-consumer
# 5688 - worker-pg-scheduler
services:
#########################################################################################################
# Python Services with debugpy
backend:
ports:
- "5678:5678"
command: [
"uv run python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5678 .venv/bin/gunicorn
--bind 0.0.0.0:8000
--workers 1
--threads 2
--worker-class gthread
--log-level debug
--timeout 900
--graceful-timeout 5 backend.wsgi:application"
]
runner:
ports:
- "5679:5679"
command: [
"uv run python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5679 .venv/bin/gunicorn
--bind 0.0.0.0:5002
--workers 1
--threads 2
--worker-class gthread
--log-level debug
--timeout 900
--access-logfile -
--graceful-timeout 5 unstract.runner:app"
]
platform-service:
ports:
- "3001:3001"
- "5680:5680"
command: [
"uv run python -Xfrozen_modules=off -m debugpy --listen 0.0.0.0:5680 .venv/bin/gunicorn
--bind 0.0.0.0:3001
--workers 1
--threads 2
--worker-class gthread
--log-level debug
--timeout 900
--access-logfile -
--graceful-timeout 5 unstract.platform_service.run:app"
]
#########################################################################################################
# V2 Workers with debugpy
# Using --pool=threads for debugpy compatibility (prefork doesn't work well with debugpy)
worker-pg-fileproc:
user: root
ports:
- "5682:5682"
entrypoint: ["bash", "-c"]
# Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
# override REPLACES the base file's command, so a Celery invocation here
# turns the service into a RabbitMQ worker while the backend enqueues to
# Postgres — the debug stack then accepts work and runs none of it.
#
# Queue set and worker type are deliberately NOT repeated: the module reads
# WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
# environment. Restating them here is what let this file drift out of sync.
command: [
"uv run python -m debugpy --listen 0.0.0.0:5682 -m pg_queue_consumer"
]
worker-pg-callback:
user: root
ports:
- "5683:5683"
entrypoint: ["bash", "-c"]
# Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
# override REPLACES the base file's command, so a Celery invocation here
# turns the service into a RabbitMQ worker while the backend enqueues to
# Postgres — the debug stack then accepts work and runs none of it.
#
# Queue set and worker type are deliberately NOT repeated: the module reads
# WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
# environment. Restating them here is what let this file drift out of sync.
command: [
"uv run python -m debugpy --listen 0.0.0.0:5683 -m pg_queue_consumer"
]
worker-pg-orchestrator-api:
user: root
ports:
- "5684:5684"
entrypoint: ["bash", "-c"]
# Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
# override REPLACES the base file's command, so a Celery invocation here
# turns the service into a RabbitMQ worker while the backend enqueues to
# Postgres — the debug stack then accepts work and runs none of it.
#
# Queue set and worker type are deliberately NOT repeated: the module reads
# WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
# environment. Restating them here is what let this file drift out of sync.
command: [
"uv run python -m debugpy --listen 0.0.0.0:5684 -m pg_queue_consumer"
]
worker-pg-orchestrator-general:
user: root
ports:
- "5685:5685"
entrypoint: ["bash", "-c"]
# Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
# override REPLACES the base file's command, so a Celery invocation here
# turns the service into a RabbitMQ worker while the backend enqueues to
# Postgres — the debug stack then accepts work and runs none of it.
#
# Queue set and worker type are deliberately NOT repeated: the module reads
# WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
# environment. Restating them here is what let this file drift out of sync.
command: [
"uv run python -m debugpy --listen 0.0.0.0:5685 -m pg_queue_consumer"
]
worker-pg-notification:
user: root
ports:
- "5686:5686"
entrypoint: ["bash", "-c"]
# Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
# override REPLACES the base file's command, so a Celery invocation here
# turns the service into a RabbitMQ worker while the backend enqueues to
# Postgres — the debug stack then accepts work and runs none of it.
#
# Queue set and worker type are deliberately NOT repeated: the module reads
# WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
# environment. Restating them here is what let this file drift out of sync.
command: [
"uv run python -m debugpy --listen 0.0.0.0:5686 -m pg_queue_consumer"
]
worker-log-stream-consumer:
user: root
ports:
- "5687:5687"
entrypoint: ["bash", "-c"]
# Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
# override REPLACES the base file's command, so a Celery invocation here
# turns the service into a RabbitMQ worker while the backend enqueues to
# Postgres — the debug stack then accepts work and runs none of it.
#
# Queue set and worker type are deliberately NOT repeated: the module reads
# WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
# environment. Restating them here is what let this file drift out of sync.
command: [
"uv run python -m debugpy --listen 0.0.0.0:5687 -m log_consumer.redis_stream_consumer"
]
worker-pg-scheduler:
user: root
ports:
- "5688:5688"
entrypoint: ["bash", "-c"]
# Wrap the PG consumer, NOT `celery -A worker worker` (UN-4046). This
# override REPLACES the base file's command, so a Celery invocation here
# turns the service into a RabbitMQ worker while the backend enqueues to
# Postgres — the debug stack then accepts work and runs none of it.
#
# Queue set and worker type are deliberately NOT repeated: the module reads
# WORKER_PG_QUEUE_CONSUMER_QUEUE / _WORKER_TYPE from the base file's
# environment. Restating them here is what let this file drift out of sync.
command: [
"uv run python -m debugpy --listen 0.0.0.0:5688 -m pg_queue_consumer"
]