LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
name: ${COMPOSE_PROJECT_NAME:-docker}
include:
- docker-compose-dev-essentials.yaml
# Reusable host-gateway mapping so containers can reach services on the host
# (e.g. host-installed Ollama at http://host.docker.internal:11434).
# NOTE: YAML merge (<<:) merges mappings shallowly and does NOT concatenate
# lists. If a service ever needs additional extra_hosts entries alongside
# this one, inline ALL entries under that service (including
# host.docker.internal:host-gateway) instead of using the anchor — a sibling
# extra_hosts key would silently shadow the anchor's list.
x-host-gateway: &host_gateway
extra_hosts:
- "host.docker.internal:host-gateway"
services:
# Backend service
backend:
<<: *host_gateway
image: unstract/backend:${VERSION}
container_name: unstract-backend
restart: unless-stopped
command: --migrate
ports:
- "8000:8000"
env_file:
- ../backend/.env
depends_on:
db:
condition: service_started
redis:
condition: service_started
rabbitmq:
condition: service_started
reverse-proxy:
condition: service_started
minio:
condition: service_started
# One-shot bucket setup. Waiting for completion keeps `up --wait` from
# treating its exit as a stack failure.
minio-bootstrap:
condition: service_completed_successfully
platform-service:
condition: service_started
x2text-service:
condition: service_started
volumes:
- prompt_studio_data:/app/prompt-studio-data
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
environment:
- APPLICATION_NAME=unstract-backend
# Redis log transport (UN-4046) — set on EVERY publisher: backend, runner
# and all twelve workers. worker-log-consumer-v2 is gone from this stack, so
# the "celery" default publishes execution logs to a RabbitMQ queue nothing
# drains; LogPublisher swallows its errors, so the only symptom is execution
# logs that stay empty while the queue grows.
#
# Deliberately here and not in essentials.env: that file is git-ignored and
# generated, and copy_or_merge_envs only merges new keys on --only-env or
# --update, so an existing checkout would silently keep the old default.
- LOG_TRANSPORT=redis
# Adopt the non-pipeline periodics (dashboard_metrics.*) onto the PG
# scheduler (UN-4046). converge_pg_scheduler only adopts them with
# --periodics, which entrypoint.sh derives from this variable — and it was
# set NOWHERE. This stack deletes celery-beat and worker-metrics, so
# without it those rows stay Beat-owned (pg_owned=false) with Beat gone:
# nothing fires them, worker-pg-metrics idles on an empty queue, and there
# is no error, no queue depth and no failed pod. The cloud chart refuses
# this combination at render time; compose has no equivalent guard.
- PG_SCHEDULER_ADOPT_PERIODICS=true
labels:
- traefik.enable=true
- traefik.http.routers.backend.rule=Host(`frontend.unstract.localhost`) && (PathPrefix(`/api/v1`) || PathPrefix(`/deployment`) || PathPrefix(`/public`))
- traefik.http.services.backend.loadbalancer.server.port=8000
# Celery Flower
celery-flower:
<<: *host_gateway
image: unstract/backend:${VERSION}
container_name: unstract-celery-flower
restart: unless-stopped
entrypoint: .venv/bin/celery
command: "-A backend flower --port=5555 --purge_offline_workers=5"
env_file:
- ../backend/.env
depends_on:
- rabbitmq
labels:
- traefik.enable=false
ports:
- "5555:5555"
environment:
- APPLICATION_NAME=unstract-celery-flower
volumes:
- unstract_data:/data
profiles:
- optional
frontend:
image: unstract/frontend:${VERSION}
container_name: unstract-frontend
restart: unless-stopped
ports:
- "3000:80"
depends_on:
- backend
- reverse-proxy
environment:
# Running platform version, surfaced on the profile page. Reuses the same
# ${VERSION} that tags the image, so it always matches what's deployed
# (and updates on RC->stable promotion, which only retags — doesn't rebuild).
- UNSTRACT_APPS_VERSION=${VERSION}
labels:
- traefik.enable=true
- traefik.http.routers.frontend.rule=Host(`frontend.unstract.localhost`) && !PathPrefix(`/api/v1`) && !PathPrefix(`/deployment`) && !PathPrefix(`/public`)
- traefik.http.services.frontend.loadbalancer.server.port=80
platform-service:
image: unstract/platform-service:${VERSION}
container_name: unstract-platform-service
restart: unless-stopped
ports:
- "3001:3001"
env_file:
- ../platform-service/.env
depends_on:
- redis
- db
labels:
- traefik.enable=false
x2text-service:
image: unstract/x2text-service:${VERSION}
container_name: unstract-x2text-service
restart: unless-stopped
ports:
- "3004:3004"
env_file:
- ../x2text-service/.env
depends_on:
- db
labels:
- traefik.enable=false
runner:
<<: *host_gateway
image: unstract/runner:${VERSION}
container_name: unstract-runner
restart: unless-stopped
ports:
- 5002:5002
env_file:
- ../runner/.env
environment:
# Publisher side of the Redis log transport (UN-4046) — see the note on the
# backend service. runner matters most of the three: it forwards this to every
# tool sidecar it spawns, so a stale value here strands the tool logs too.
- LOG_TRANSPORT=redis
volumes:
- ./workflow_data:/data
# Docker socket bind mount to spawn tool containers
- /var/run/docker.sock:/var/run/docker.sock
depends_on:
- redis
- rabbitmq
labels:
- traefik.enable=false
worker-log-history-scheduler-v2:
<<: *host_gateway
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-log-history-scheduler-v2
restart: unless-stopped
entrypoint: ["/bin/bash"]
command: ["/app/log_consumer/scheduler.sh"]
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
- rabbitmq
environment:
- APPLICATION_NAME=unstract-worker-log-history-scheduler-v2
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# Scheduler interval in seconds
- LOG_HISTORY_CONSUMER_INTERVAL=${LOG_HISTORY_CONSUMER_INTERVAL:-5}
# Override example: TASK_TRIGGER_COMMAND=/custom/trigger/script.sh
- TASK_TRIGGER_COMMAND=${TASK_TRIGGER_COMMAND:-}
labels:
- traefik.enable=false
worker-pg-orchestrator-api:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-orchestrator-api
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8093:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-orchestrator-api
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=api_deployment
- WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_api_deployments
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# async_execute_bin arms the barrier + dispatches batches, then returns — a
# bounded-minutes orchestration, and idempotency-guarded (UN-3671) so a vt
# overrun can't double-orchestrate. Without an explicit vt the consumer
# defaults to 30s and a slow dispatch redelivers mid-run. 600/660 mirrors the
# k8s deployment defaults (grace-equivalent handled by restart:unless-stopped here).
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_ORCHESTRATOR_VT_SECONDS:-600}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_ORCHESTRATOR_HEALTH_STALE_SECONDS:-660}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
# Orchestrator consumer for general / scheduled-ETL executions (async_execute_bin).
worker-pg-orchestrator-general:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-orchestrator-general
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8094:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-orchestrator-general
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=general
- WORKER_PG_QUEUE_CONSUMER_QUEUE=celery
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# Orchestration is bounded-minutes + claim-guarded (see orchestrator-api).
# Without an explicit vt it defaults to 30s. 600/660 matches our k8s deployment defaults (workerPg* values, separate chart repo).
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_ORCHESTRATOR_VT_SECONDS:-600}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_ORCHESTRATOR_HEALTH_STALE_SECONDS:-660}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
# Fan-out consumer (process_file_batch) — both the ETL and API file queues.
worker-pg-fileproc:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-fileproc
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8095:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
- rabbitmq # still dispatches the tool-execution RPC to the Celery executor (until the executor/tool-RPC moves to PG)
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-fileproc
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=file_processing
- WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,file_processing_priority,api_file_processing
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# UN-3606: prefork N consumer children so file batches run in parallel (the
# PG analogue of the Celery worker's --concurrency). SKIP LOCKED distributes
# batches across children + replicas; a single execution is still capped by
# MAX_PARALLEL_FILE_BATCHES. Mirrors WORKER_FILE_PROCESSING_CONCURRENCY.
- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_FILEPROC_CONCURRENCY:-4}
# process_file_batch blocks on the executor RPC (up to EXECUTOR_RESULT_TIMEOUT,
# default 3600s). vt must exceed ONE batch's worst case (children run in
# parallel, not serially) or a long batch gets re-claimed mid-run; health-stale
# sits just above vt so a legitimately-long batch never trips the fleet probe.
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_FILEPROC_VT_SECONDS:-3660}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_FILEPROC_HEALTH_STALE_SECONDS:-3720}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
# Fan-in callback consumer (process_batch_callback / _api) — both callback queues.
worker-pg-callback:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-callback
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8096:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
- rabbitmq # may dispatch Celery notifications when not PG-routed (until notifications move to PG)
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-callback
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=callback
- WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing_callback,celery_callback,api_file_processing_callback
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# The aggregating callback (fan-in) runs after all batches: aggregate results,
# update status, push destination, notify. Without an explicit vt it defaults
# to 30s and a >30s callback redelivers — re-firing webhooks + subscription
# billing. 3660/3720 matches our k8s deployment defaults (separate chart repo).
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_CALLBACK_VT_SECONDS:-3660}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_CALLBACK_HEALTH_STALE_SECONDS:-3720}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
# Scheduler consumer — runs scheduler.tasks.execute_pipeline_task fired by the
# orchestrator's PG scheduler tick (Beat replacement). Distinct from the Celery
# 'scheduler' worker; this only runs the trigger, which then creates the
# execution + dispatches the workflow exactly as today.
worker-pg-scheduler:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-scheduler
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8098:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
- rabbitmq # the trigger dispatches async_execute_bin (Celery path when not PG-routed)
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-scheduler
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler
- WORKER_PG_QUEUE_CONSUMER_QUEUE=scheduler
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# Scheduled-trigger dispatch is quick (enqueue the execution). Without an
# explicit vt it defaults to 30s. 180/240 matches our k8s deployment defaults (separate chart repo).
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_SCHEDULER_VT_SECONDS:-180}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_SCHEDULER_HEALTH_STALE_SECONDS:-240}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
# Dashboard-metrics periodics over PG (UN-3796) — the execution half of the Beat
# replacement. The PG scheduler fires dashboard_metrics.* onto dashboard_metric_events
# and this drains them, replacing the Celery worker-metrics
# (-A backend worker -Q dashboard_metric_events).
#
# A dedicated service rather than another queue on worker-pg-scheduler: the
# consumer's health heartbeat is stamped per poll and FROZEN while a task runs, so
# HEALTH_STALE doubles as an upper bound on one task's wall clock. The aggregation
# runs for minutes; under the scheduler's 240s bound it would trip the liveness
# probe, restart the pod, and take in-flight pipeline triggers down with it.
#
# No rabbitmq dependency — the tasks are thin HTTP proxies to the backend internal
# API and dispatch nothing onward. MAX_ATTEMPTS=1 because for a periodic the next
# cron tick supersedes a failed run; retrying would let a persistently failing
# aggregation stack up in-flight messages.
worker-pg-metrics:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-metrics
restart: unless-stopped
# The GENERIC consumer command, with identity carried by the env below — the
# container entrypoint (run-worker-docker.sh) is env-driven and its `pg-*`
# dispatch accepts only pg-queue-consumer / pg-consumer / pg-queue-reaper /
# pg-reaper / reaper, rejecting anything else with exit 1. The named
# `pg-metrics` role is for the HOST runner (`./run-worker.sh pg-metrics`),
# which is a different script and does understand roles.
command: ["pg-queue-consumer"]
ports:
- "8101:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-metrics
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler
- WORKER_PG_QUEUE_CONSUMER_QUEUE=dashboard_metric_events
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# Sized for a minutes-long aggregation: stale > vt > the task's own ceiling.
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_METRICS_VT_SECONDS:-900}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_METRICS_HEALTH_STALE_SECONDS:-960}
# One at a time. Since UN-3974 split the aggregation by tier the crons can
# overlap (the :20 daily/monthly row against a long */15 run, and the 04:40
# reconcile against the 04:45 one), and the per-tier lock keys deliberately do
# not exclude them — so this serialises them instead. Raise to 2 if the tail
# delay matters more than the DB load two concurrent runs add.
- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=1
- WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS=1
labels:
- traefik.enable=false
# Log streaming without Celery (UN-3755) — the flag-on replacement for
# the retired Celery log consumer. Note the transport here is a REDIS LIST, not
# pg_queue_message: the log consumer's own sinks are already Redis (log_history_queue
# + the Socket.IO emit), so routing through Postgres would detour a lossy, high-rate
# stream through the platform's most contended resource to buy durability that logs
# explicitly discard at their 10k cap. Despite the name it speaks to Redis, not
# Postgres; it is part of the PG-era fleet by association, not by transport.
#
# LOG_TRANSPORT must be set to "redis" on every PUBLISHER too or this drains an
# empty list while logs keep going to RabbitMQ — where, since UN-4046 removed
# worker-log-consumer-v2 from this stack, nothing reads them. Set on backend and
# runner in their own environment blocks, and in each worker's own environment
# block too — deliberately NOT essentials.env (that file is git-ignored and
# generated, so an existing checkout only picks up new keys on --update).
worker-log-stream-consumer:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-log-stream-consumer
restart: unless-stopped
command: ["log-stream-consumer"]
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- redis
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-log-stream-consumer
- WORKER_TYPE=log_consumer
- WORKER_NAME=log-stream-consumer
- LOG_TRANSPORT=redis
- LOG_STREAM_QUEUE_NAME=${LOG_STREAM_QUEUE_NAME:-log_stream_queue}
# Downstream sinks are unchanged — same durable buffer the Celery worker wrote to.
- LOG_HISTORY_QUEUE_NAME=${LOG_HISTORY_QUEUE_NAME:-log_history_queue}
- ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}
labels:
- traefik.enable=false
# Executor RPC over PG — runs execute_extraction as a request-reply: claims
# work from Postgres and writes the ExecutionResult to pg_task_result for the
# blocking caller. Same heavy executor runtime as the file-processing consumer (tool
# execution → platform-service / adapters / file storage), but broker-free: it
# is the terminal of the RPC and dispatches nothing onward to Celery. It is the
# ONLY executor since UN-4046 — scaling it to zero stops all tool execution.
worker-pg-executor:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-executor
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8099:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
- platform-service
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-executor
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=executor
- WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# The executor runs LLM inference — far longer than the consumer's 30s/60s
# defaults. Set the visibility timeout AND the liveness stale-threshold
# ABOVE the executor's hard per-task limit (EXECUTOR_TASK_TIME_LIMIT=3600s),
# else a long task is re-claimed mid-run (double execution / token
# double-spend, two runs racing the same reply_key) or killed by the probe.
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_EXECUTOR_VT_SECONDS:-3660}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_EXECUTOR_HEALTH_STALE_SECONDS:-3720}
# UN-3606: prefork N executor children so concurrent file batches' executor
# RPCs run in parallel instead of serializing here. Without this the
# file-processing parallelism delivers no end-to-end speedup — the LLM
# extraction (the wall-clock-dominant work) is the bottleneck. Match the
# file-processing concurrency so N parallel files get N parallel executors.
- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_EXECUTOR_CONCURRENCY:-4}
- CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- prompt_studio_data:/app/prompt-studio-data
# IDE callback consumer (③c) — drains the ``ide_callback`` queue the executor
# self-chains onto for async/callback dispatch (Prompt Studio run/index/extract,
# lookups). Registers the ide_callback worker's tasks (ide_prompt_complete,
# ide_index_complete, extraction_complete, …). Idle (receives no work) until
# dispatch_with_callback routes to PG (i.e. the gating flag is on); the Celery
# ``worker-ide-callback`` still serves the Celery path. No broker dependency:
# these callbacks only do API writes + ws emits, never an onward Celery dispatch.
# Lightweight, so a modest concurrency suffices; scale via replicas (SKIP LOCKED).
worker-pg-ide-callback:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-ide-callback
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8100:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
- platform-service
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-ide-callback
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=ide_callback
# agent_kv_callback carries the Agent-KV terminal callbacks (finalize:
# persist the result, delete the staged input, release the concurrency
# slot, fire the webhook). Without it a job runs to success and then
# sits in RUNNING forever -- the executor's continuation is enqueued
# and nothing drains it. The Celery ide_callback worker has carried
# both queues since the callbacks landed; its PG twin did not.
- WORKER_PG_QUEUE_CONSUMER_QUEUE=ide_callback,agent_kv_callback
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_IDE_CALLBACK_CONCURRENCY:-2}
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# Callbacks are sub-second (ws emit + small API writes), so nowhere near the
# executor's 1h limit — but set the visibility timeout above a slow internal
# API write (backend under load) so a sibling replica can't re-claim mid-run
# and double-emit. Health-stale sits above it. Overridable per-env.
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${PG_IDE_CALLBACK_VT_SECONDS:-120}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${PG_IDE_CALLBACK_HEALTH_STALE_SECONDS:-180}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- prompt_studio_data:/app/prompt-studio-data
# Buffered webhook notifications. The backend's notification seam enqueues
# ``send_webhook_notification`` onto these queues unconditionally (UN-4046), so
# without this consumer every buffered webhook would sit undrained — nothing
# sweeps pg_queue_message.
worker-pg-notification:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-notification
restart: unless-stopped
command: ["pg-queue-consumer"]
ports:
- "8104:8090"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
- platform-service
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-notification
- WORKER_BARRIER_BACKEND=pg
- WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification
- WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority
- WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
- WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_NOTIFICATION_CONCURRENCY:-2}
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# One outbound HTTP POST per task. VT sits above a slow subscriber so a
# sibling replica cannot re-claim mid-POST and double-deliver; health-stale
# above that. Mirrors the chart's workerPgNotification bounds.
- WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${PG_NOTIFICATION_VT_SECONDS:-300}
- WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${PG_NOTIFICATION_HEALTH_STALE_SECONDS:-360}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
- prompt_studio_data:/app/prompt-studio-data
# Reaper / orchestrator — leader-elected loop. Run exactly ONE instance (it
# elects a single leader via pg_orchestrator_lock; extra replicas idle as
# standby). Besides barrier-orphan recovery it runs the PG scheduler tick
# (fires due pg_owned schedules — Beat replacement); dark until rows are owned.
worker-pg-reaper:
image: unstract/worker-unified:${VERSION}
container_name: unstract-worker-pg-reaper
restart: unless-stopped
command: ["pg-queue-reaper"]
ports:
- "8097:8086"
env_file:
- ../workers/.env
- ./essentials.env
depends_on:
- db
- redis
environment:
- ENVIRONMENT=development
- APPLICATION_NAME=unstract-worker-pg-reaper
- WORKER_PG_REAPER_INTERVAL_SECONDS=${WORKER_PG_REAPER_INTERVAL_SECONDS:-5}
- WORKER_PG_REAPER_HEALTH_PORT=8086
# Log publisher — see the note on the backend service (UN-4046).
- LOG_TRANSPORT=redis
# Retention sweep (cadence-gated inside the leader's tick): drop expired
# pg_task_result rows + orphaned pg_batch_dedup markers so neither grows
# unbounded as the gate ramps. The dedup retention below (default 24h) should
# be kept above the longest possible execution, so an in-flight marker is
# never swept.
- WORKER_PG_REAPER_SWEEP_SECONDS=${WORKER_PG_REAPER_SWEEP_SECONDS:-300}
- WORKER_PG_DEDUP_RETENTION_SECONDS=${WORKER_PG_DEDUP_RETENTION_SECONDS:-86400}
labels:
- traefik.enable=false
volumes:
- ./workflow_data:/data
- ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
volumes:
prompt_studio_data:
unstract_data:
networks:
default:
# NOTE:
# Any changes need to be reflected in proxy service too.
name: unstract-network