LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
# Development override - NO debugpy (use compose.debug.yaml for debugging)
#
# Usage:
# Normal dev: VERSION=test docker compose -f docker-compose.yaml -f compose.override.yaml watch
# With debug: VERSION=test docker compose -f docker-compose.yaml -f compose.override.yaml -f compose.debug.yaml watch
#
# Memory-optimized settings: reduced workers/threads for local development
#
# Requirements:
# - Docker Compose 2.24.4+ (for !override directive support)
services:
#########################################################################################################
# Frontend - React dev server with hot reload
frontend:
image: unstract/frontend:${VERSION}
build:
dockerfile: docker/dockerfiles/frontend.Dockerfile
context: ..
target: development
depends_on: !override
- reverse-proxy
env_file:
- ../frontend/.env
develop:
watch:
- action: sync
path: ../frontend/
target: /app
ignore:
- node_modules/
- build/
- action: rebuild
path: ../frontend/bun.lock
- action: rebuild
path: ../frontend/package.json
labels:
- traefik.http.services.frontend.loadbalancer.server.port=3000
#########################################################################################################
# Backend - Gunicorn with reload (memory optimized: 1 worker, 2 threads)
backend:
image: unstract/backend:${VERSION}
build:
dockerfile: docker/dockerfiles/backend.Dockerfile
context: ..
depends_on: !override
- db
- redis
- reverse-proxy
- minio
- platform-service
init: true
entrypoint: ["bash", "-c"]
command: [
"uv run python -Xfrozen_modules=off .venv/bin/gunicorn
--bind 0.0.0.0:8000
--workers 1
--threads 2
--worker-class gthread
--log-level debug
--timeout 900
--access-logfile -
--graceful-timeout 5 backend.wsgi:application"
]
develop:
watch:
- action: sync+restart
path: ../backend/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
labels:
- traefik.http.services.backend.loadbalancer.server.port=8000
#########################################################################################################
# Runner (memory optimized: 1 worker, 2 threads)
runner:
image: unstract/runner:${VERSION}
build:
dockerfile: docker/dockerfiles/runner.Dockerfile
context: ..
entrypoint: ["bash", "-c"]
command: [
"uv run python -Xfrozen_modules=off .venv/bin/gunicorn
--bind 0.0.0.0:5002
--workers 1
--threads 2
--worker-class gthread
--log-level debug
--timeout 900
--access-logfile -
--graceful-timeout 5 unstract.runner:app"
]
develop:
watch:
- action: sync+restart
path: ../runner/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../runner/uv.lock
#########################################################################################################
# Platform Service (memory optimized: 1 worker, 2 threads)
platform-service:
image: unstract/platform-service:${VERSION}
build:
dockerfile: docker/dockerfiles/platform.Dockerfile
context: ..
ports:
- "3001:3001"
entrypoint: ["bash", "-c"]
command: [
"uv run python -Xfrozen_modules=off .venv/bin/gunicorn
--bind 0.0.0.0:3001
--workers 1
--threads 2
--worker-class gthread
--log-level debug
--timeout 900
--access-logfile -
--graceful-timeout 5 unstract.platform_service.run:app"
]
develop:
watch:
- action: sync+restart
path: ../platform-service/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../platform-service/uv.lock
#########################################################################################################
# X2Text Service
x2text-service:
image: unstract/x2text-service:${VERSION}
profiles:
- optional
build:
dockerfile: docker/dockerfiles/x2text.Dockerfile
context: ..
#########################################################################################################
# Tool Sidecar
tool-sidecar:
image: unstract/tool-sidecar:${VERSION}
profiles:
- tools
build:
dockerfile: docker/dockerfiles/tool-sidecar.Dockerfile
context: ..
#########################################################################################################
# V2 Workers - Build configurations with memory-optimized settings
# All use --pool=threads and --concurrency=2 for lower memory footprint
worker-pg-orchestrator-api:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../workers/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../workers/uv.lock
worker-pg-callback:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../workers/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../workers/uv.lock
worker-pg-fileproc:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../workers/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../workers/uv.lock
worker-pg-orchestrator-general:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../workers/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../workers/uv.lock
worker-pg-notification:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../workers/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../workers/uv.lock
worker-log-stream-consumer:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../workers/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../workers/uv.lock
# No develop.watch - runs bash scheduler script, not Python code
worker-log-history-scheduler-v2:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
worker-pg-scheduler:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../workers/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: rebuild
path: ../workers/uv.lock
#########################################################################################################
# IDE callback worker (workers image, processes ide_callback queue)
worker-pg-ide-callback:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
# PG consumer for the dashboard_metrics.* periodics fired by the PG scheduler
# tick. Must build the WORKER image: the base file runs `pg-queue-consumer`,
# which the backend image's entrypoint rejects outright ("Unknown argument"),
# so building backend.Dockerfile here crash-loops the pod and leaves
# dashboard_metric_events with no consumer. No `command:` override either —
# the base file's is correct.
worker-pg-metrics:
build:
dockerfile: docker/dockerfiles/worker-unified.Dockerfile
context: ..
develop:
watch:
- action: sync+restart
path: ../backend/
target: /app
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
- action: sync+restart
path: ../unstract/
target: /unstract
ignore: [.venv/, __pycache__/, "*.pyc", .pytest_cache/, .mypy_cache/]
# V1 workers disabled by default (use workers-v2 profile instead)
worker:
profiles:
- workers-v1
worker-file-processing:
profiles:
- workers-v1
worker-file-processing-callback:
profiles:
- workers-v1
worker-logging:
profiles:
- workers-v1