Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

28.2 KB
Raw Blame
name: ${COMPOSE_PROJECT_NAME:-docker}
include:
  - docker-compose-dev-essentials.yaml

# Reusable host-gateway mapping so containers can reach services on the host
# (e.g. host-installed Ollama at http://host.docker.internal:11434).
# NOTE: YAML merge (<<:) merges mappings shallowly and does NOT concatenate
# lists. If a service ever needs additional extra_hosts entries alongside
# this one, inline ALL entries under that service (including
# host.docker.internal:host-gateway) instead of using the anchor — a sibling
# extra_hosts key would silently shadow the anchor's list.
x-host-gateway: &host_gateway
  extra_hosts:
    - "host.docker.internal:host-gateway"

services:
  # Backend service
  backend:
    <<: *host_gateway
    image: unstract/backend:${VERSION}
    container_name: unstract-backend
    restart: unless-stopped
    command: --migrate
    ports:
      - "8000:8000"
    env_file:
      - ../backend/.env
    depends_on:
      db:
        condition: service_started
      redis:
        condition: service_started
      rabbitmq:
        condition: service_started
      reverse-proxy:
        condition: service_started
      minio:
        condition: service_started
      # One-shot bucket setup. Waiting for completion keeps `up --wait` from
      # treating its exit as a stack failure.
      minio-bootstrap:
        condition: service_completed_successfully
      platform-service:
        condition: service_started
      x2text-service:
        condition: service_started
    volumes:
      - prompt_studio_data:/app/prompt-studio-data
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
    environment:
      - APPLICATION_NAME=unstract-backend
      # Redis log transport (UN-4046) — set on EVERY publisher: backend, runner
      # and all twelve workers. worker-log-consumer-v2 is gone from this stack, so
      # the "celery" default publishes execution logs to a RabbitMQ queue nothing
      # drains; LogPublisher swallows its errors, so the only symptom is execution
      # logs that stay empty while the queue grows.
      #
      # Deliberately here and not in essentials.env: that file is git-ignored and
      # generated, and copy_or_merge_envs only merges new keys on --only-env or
      # --update, so an existing checkout would silently keep the old default.
      - LOG_TRANSPORT=redis
      # Adopt the non-pipeline periodics (dashboard_metrics.*) onto the PG
      # scheduler (UN-4046). converge_pg_scheduler only adopts them with
      # --periodics, which entrypoint.sh derives from this variable — and it was
      # set NOWHERE. This stack deletes celery-beat and worker-metrics, so
      # without it those rows stay Beat-owned (pg_owned=false) with Beat gone:
      # nothing fires them, worker-pg-metrics idles on an empty queue, and there
      # is no error, no queue depth and no failed pod. The cloud chart refuses
      # this combination at render time; compose has no equivalent guard.
      - PG_SCHEDULER_ADOPT_PERIODICS=true
    labels:
      - traefik.enable=true
      - traefik.http.routers.backend.rule=Host(`frontend.unstract.localhost`) && (PathPrefix(`/api/v1`) || PathPrefix(`/deployment`) || PathPrefix(`/public`))
      - traefik.http.services.backend.loadbalancer.server.port=8000

  # Celery Flower
  celery-flower:
    <<: *host_gateway
    image: unstract/backend:${VERSION}
    container_name: unstract-celery-flower
    restart: unless-stopped
    entrypoint: .venv/bin/celery
    command: "-A backend flower --port=5555 --purge_offline_workers=5"
    env_file:
      - ../backend/.env
    depends_on:
      - rabbitmq
    labels:
      - traefik.enable=false
    ports:
      - "5555:5555"
    environment:
      - APPLICATION_NAME=unstract-celery-flower
    volumes:
      - unstract_data:/data
    profiles:
      - optional

  frontend:
    image: unstract/frontend:${VERSION}
    container_name: unstract-frontend
    restart: unless-stopped
    ports:
      - "3000:80"
    depends_on:
      - backend
      - reverse-proxy
    environment:
      # Running platform version, surfaced on the profile page. Reuses the same
      # ${VERSION} that tags the image, so it always matches what's deployed
      # (and updates on RC->stable promotion, which only retags — doesn't rebuild).
      - UNSTRACT_APPS_VERSION=${VERSION}
    labels:
      - traefik.enable=true
      - traefik.http.routers.frontend.rule=Host(`frontend.unstract.localhost`) && !PathPrefix(`/api/v1`) && !PathPrefix(`/deployment`) && !PathPrefix(`/public`)
      - traefik.http.services.frontend.loadbalancer.server.port=80

  platform-service:
    image: unstract/platform-service:${VERSION}
    container_name: unstract-platform-service
    restart: unless-stopped
    ports:
      - "3001:3001"
    env_file:
      - ../platform-service/.env
    depends_on:
      - redis
      - db
    labels:
      - traefik.enable=false

  x2text-service:
    image: unstract/x2text-service:${VERSION}
    container_name: unstract-x2text-service
    restart: unless-stopped
    ports:
      - "3004:3004"
    env_file:
      - ../x2text-service/.env
    depends_on:
      - db
    labels:
      - traefik.enable=false

  runner:
    <<: *host_gateway
    image: unstract/runner:${VERSION}
    container_name: unstract-runner
    restart: unless-stopped
    ports:
      - 5002:5002
    env_file:
      - ../runner/.env
    environment:
      # Publisher side of the Redis log transport (UN-4046) — see the note on the
      # backend service. runner matters most of the three: it forwards this to every
      # tool sidecar it spawns, so a stale value here strands the tool logs too.
      - LOG_TRANSPORT=redis
    volumes:
      - ./workflow_data:/data
      # Docker socket bind mount to spawn tool containers
      - /var/run/docker.sock:/var/run/docker.sock
    depends_on:
      - redis
      - rabbitmq
    labels:
      - traefik.enable=false

  worker-log-history-scheduler-v2:
    <<: *host_gateway
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-log-history-scheduler-v2
    restart: unless-stopped
    entrypoint: ["/bin/bash"]
    command: ["/app/log_consumer/scheduler.sh"]
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
      - rabbitmq
    environment:
      - APPLICATION_NAME=unstract-worker-log-history-scheduler-v2
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # Scheduler interval in seconds
      - LOG_HISTORY_CONSUMER_INTERVAL=${LOG_HISTORY_CONSUMER_INTERVAL:-5}
      # Override example: TASK_TRIGGER_COMMAND=/custom/trigger/script.sh
      - TASK_TRIGGER_COMMAND=${TASK_TRIGGER_COMMAND:-}
    labels:
      - traefik.enable=false

  worker-pg-orchestrator-api:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-orchestrator-api
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8093:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-orchestrator-api
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=api_deployment
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_api_deployments
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # async_execute_bin arms the barrier + dispatches batches, then returns — a
      # bounded-minutes orchestration, and idempotency-guarded (UN-3671) so a vt
      # overrun can't double-orchestrate. Without an explicit vt the consumer
      # defaults to 30s and a slow dispatch redelivers mid-run. 600/660 mirrors the
      # k8s deployment defaults (grace-equivalent handled by restart:unless-stopped here).
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_ORCHESTRATOR_VT_SECONDS:-600}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_ORCHESTRATOR_HEALTH_STALE_SECONDS:-660}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config

  # Orchestrator consumer for general / scheduled-ETL executions (async_execute_bin).
  worker-pg-orchestrator-general:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-orchestrator-general
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8094:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-orchestrator-general
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=general
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # Orchestration is bounded-minutes + claim-guarded (see orchestrator-api).
      # Without an explicit vt it defaults to 30s. 600/660 matches our k8s deployment defaults (workerPg* values, separate chart repo).
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_ORCHESTRATOR_VT_SECONDS:-600}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_ORCHESTRATOR_HEALTH_STALE_SECONDS:-660}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config

  # Fan-out consumer (process_file_batch) — both the ETL and API file queues.
  worker-pg-fileproc:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-fileproc
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8095:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
      - rabbitmq   # still dispatches the tool-execution RPC to the Celery executor (until the executor/tool-RPC moves to PG)
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-fileproc
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=file_processing
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing,file_processing_priority,api_file_processing
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # UN-3606: prefork N consumer children so file batches run in parallel (the
      # PG analogue of the Celery worker's --concurrency). SKIP LOCKED distributes
      # batches across children + replicas; a single execution is still capped by
      # MAX_PARALLEL_FILE_BATCHES. Mirrors WORKER_FILE_PROCESSING_CONCURRENCY.
      - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_FILEPROC_CONCURRENCY:-4}
      # process_file_batch blocks on the executor RPC (up to EXECUTOR_RESULT_TIMEOUT,
      # default 3600s). vt must exceed ONE batch's worst case (children run in
      # parallel, not serially) or a long batch gets re-claimed mid-run; health-stale
      # sits just above vt so a legitimately-long batch never trips the fleet probe.
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_FILEPROC_VT_SECONDS:-3660}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_FILEPROC_HEALTH_STALE_SECONDS:-3720}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config

  # Fan-in callback consumer (process_batch_callback / _api) — both callback queues.
  worker-pg-callback:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-callback
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8096:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
      - rabbitmq   # may dispatch Celery notifications when not PG-routed (until notifications move to PG)
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-callback
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=callback
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=file_processing_callback,celery_callback,api_file_processing_callback
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # The aggregating callback (fan-in) runs after all batches: aggregate results,
      # update status, push destination, notify. Without an explicit vt it defaults
      # to 30s and a >30s callback redelivers — re-firing webhooks + subscription
      # billing. 3660/3720 matches our k8s deployment defaults (separate chart repo).
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_CALLBACK_VT_SECONDS:-3660}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_CALLBACK_HEALTH_STALE_SECONDS:-3720}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config

  # Scheduler consumer — runs scheduler.tasks.execute_pipeline_task fired by the
  # orchestrator's PG scheduler tick (Beat replacement). Distinct from the Celery
  # 'scheduler' worker; this only runs the trigger, which then creates the
  # execution + dispatches the workflow exactly as today.
  worker-pg-scheduler:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-scheduler
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8098:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
      - rabbitmq   # the trigger dispatches async_execute_bin (Celery path when not PG-routed)
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-scheduler
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=scheduler
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # Scheduled-trigger dispatch is quick (enqueue the execution). Without an
      # explicit vt it defaults to 30s. 180/240 matches our k8s deployment defaults (separate chart repo).
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_SCHEDULER_VT_SECONDS:-180}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_SCHEDULER_HEALTH_STALE_SECONDS:-240}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config

  # Dashboard-metrics periodics over PG (UN-3796) — the execution half of the Beat
  # replacement. The PG scheduler fires dashboard_metrics.* onto dashboard_metric_events
  # and this drains them, replacing the Celery worker-metrics
  # (-A backend worker -Q dashboard_metric_events).
  #
  # A dedicated service rather than another queue on worker-pg-scheduler: the
  # consumer's health heartbeat is stamped per poll and FROZEN while a task runs, so
  # HEALTH_STALE doubles as an upper bound on one task's wall clock. The aggregation
  # runs for minutes; under the scheduler's 240s bound it would trip the liveness
  # probe, restart the pod, and take in-flight pipeline triggers down with it.
  #
  # No rabbitmq dependency — the tasks are thin HTTP proxies to the backend internal
  # API and dispatch nothing onward. MAX_ATTEMPTS=1 because for a periodic the next
  # cron tick supersedes a failed run; retrying would let a persistently failing
  # aggregation stack up in-flight messages.
  worker-pg-metrics:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-metrics
    restart: unless-stopped
    # The GENERIC consumer command, with identity carried by the env below — the
    # container entrypoint (run-worker-docker.sh) is env-driven and its `pg-*`
    # dispatch accepts only pg-queue-consumer / pg-consumer / pg-queue-reaper /
    # pg-reaper / reaper, rejecting anything else with exit 1. The named
    # `pg-metrics` role is for the HOST runner (`./run-worker.sh pg-metrics`),
    # which is a different script and does understand roles.
    command: ["pg-queue-consumer"]
    ports:
      - "8101:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-metrics
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=scheduler
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=dashboard_metric_events
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # Sized for a minutes-long aggregation: stale > vt > the task's own ceiling.
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_METRICS_VT_SECONDS:-900}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_METRICS_HEALTH_STALE_SECONDS:-960}
      # One at a time. Since UN-3974 split the aggregation by tier the crons can
      # overlap (the :20 daily/monthly row against a long */15 run, and the 04:40
      # reconcile against the 04:45 one), and the per-tier lock keys deliberately do
      # not exclude them — so this serialises them instead. Raise to 2 if the tail
      # delay matters more than the DB load two concurrent runs add.
      - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=1
      - WORKER_PG_QUEUE_CONSUMER_MAX_ATTEMPTS=1
    labels:
      - traefik.enable=false

  # Log streaming without Celery (UN-3755) — the flag-on replacement for
  # the retired Celery log consumer. Note the transport here is a REDIS LIST, not
  # pg_queue_message: the log consumer's own sinks are already Redis (log_history_queue
  # + the Socket.IO emit), so routing through Postgres would detour a lossy, high-rate
  # stream through the platform's most contended resource to buy durability that logs
  # explicitly discard at their 10k cap. Despite the name it speaks to Redis, not
  # Postgres; it is part of the PG-era fleet by association, not by transport.
  #
  # LOG_TRANSPORT must be set to "redis" on every PUBLISHER too or this drains an
  # empty list while logs keep going to RabbitMQ — where, since UN-4046 removed
  # worker-log-consumer-v2 from this stack, nothing reads them. Set on backend and
  # runner in their own environment blocks, and in each worker's own environment
  # block too — deliberately NOT essentials.env (that file is git-ignored and
  # generated, so an existing checkout only picks up new keys on --update).
  worker-log-stream-consumer:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-log-stream-consumer
    restart: unless-stopped
    command: ["log-stream-consumer"]
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - redis
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-log-stream-consumer
      - WORKER_TYPE=log_consumer
      - WORKER_NAME=log-stream-consumer
      - LOG_TRANSPORT=redis
      - LOG_STREAM_QUEUE_NAME=${LOG_STREAM_QUEUE_NAME:-log_stream_queue}
      # Downstream sinks are unchanged — same durable buffer the Celery worker wrote to.
      - LOG_HISTORY_QUEUE_NAME=${LOG_HISTORY_QUEUE_NAME:-log_history_queue}
      - ENABLE_LOG_HISTORY=${ENABLE_LOG_HISTORY:-true}
    labels:
      - traefik.enable=false

  # Executor RPC over PG — runs execute_extraction as a request-reply: claims
  # work from Postgres and writes the ExecutionResult to pg_task_result for the
  # blocking caller. Same heavy executor runtime as the file-processing consumer (tool
  # execution → platform-service / adapters / file storage), but broker-free: it
  # is the terminal of the RPC and dispatches nothing onward to Celery. It is the
  # ONLY executor since UN-4046 — scaling it to zero stops all tool execution.
  worker-pg-executor:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-executor
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8099:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
      - platform-service
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-executor
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=executor
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # The executor runs LLM inference — far longer than the consumer's 30s/60s
      # defaults. Set the visibility timeout AND the liveness stale-threshold
      # ABOVE the executor's hard per-task limit (EXECUTOR_TASK_TIME_LIMIT=3600s),
      # else a long task is re-claimed mid-run (double execution / token
      # double-spend, two runs racing the same reply_key) or killed by the probe.
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${WORKER_PG_EXECUTOR_VT_SECONDS:-3660}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${WORKER_PG_EXECUTOR_HEALTH_STALE_SECONDS:-3720}
      # UN-3606: prefork N executor children so concurrent file batches' executor
      # RPCs run in parallel instead of serializing here. Without this the
      # file-processing parallelism delivers no end-to-end speedup — the LLM
      # extraction (the wall-clock-dominant work) is the bottleneck. Match the
      # file-processing concurrency so N parallel files get N parallel executors.
      - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_EXECUTOR_CONCURRENCY:-4}
      - CELERY_QUEUES_EXECUTOR=${CELERY_QUEUES_EXECUTOR:-celery_executor_legacy,celery_executor_agentic,celery_executor_agentic_table}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
      - prompt_studio_data:/app/prompt-studio-data

  # IDE callback consumer (③c) — drains the ``ide_callback`` queue the executor
  # self-chains onto for async/callback dispatch (Prompt Studio run/index/extract,
  # lookups). Registers the ide_callback worker's tasks (ide_prompt_complete,
  # ide_index_complete, extraction_complete, …). Idle (receives no work) until
  # dispatch_with_callback routes to PG (i.e. the gating flag is on); the Celery
  # ``worker-ide-callback`` still serves the Celery path. No broker dependency:
  # these callbacks only do API writes + ws emits, never an onward Celery dispatch.
  # Lightweight, so a modest concurrency suffices; scale via replicas (SKIP LOCKED).
  worker-pg-ide-callback:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-ide-callback
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8100:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
      - platform-service
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-ide-callback
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=ide_callback
      # agent_kv_callback carries the Agent-KV terminal callbacks (finalize:
      # persist the result, delete the staged input, release the concurrency
      # slot, fire the webhook). Without it a job runs to success and then
      # sits in RUNNING forever -- the executor's continuation is enqueued
      # and nothing drains it. The Celery ide_callback worker has carried
      # both queues since the callbacks landed; its PG twin did not.
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=ide_callback,agent_kv_callback
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_IDE_CALLBACK_CONCURRENCY:-2}
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # Callbacks are sub-second (ws emit + small API writes), so nowhere near the
      # executor's 1h limit — but set the visibility timeout above a slow internal
      # API write (backend under load) so a sibling replica can't re-claim mid-run
      # and double-emit. Health-stale sits above it. Overridable per-env.
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${PG_IDE_CALLBACK_VT_SECONDS:-120}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${PG_IDE_CALLBACK_HEALTH_STALE_SECONDS:-180}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
      - prompt_studio_data:/app/prompt-studio-data

  # Buffered webhook notifications. The backend's notification seam enqueues
  # ``send_webhook_notification`` onto these queues unconditionally (UN-4046), so
  # without this consumer every buffered webhook would sit undrained — nothing
  # sweeps pg_queue_message.
  worker-pg-notification:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-notification
    restart: unless-stopped
    command: ["pg-queue-consumer"]
    ports:
      - "8104:8090"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
      - platform-service
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-notification
      - WORKER_BARRIER_BACKEND=pg
      - WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE=notification
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT=8090
      - WORKER_PG_QUEUE_CONSUMER_CONCURRENCY=${PG_NOTIFICATION_CONCURRENCY:-2}
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # One outbound HTTP POST per task. VT sits above a slow subscriber so a
      # sibling replica cannot re-claim mid-POST and double-deliver; health-stale
      # above that. Mirrors the chart's workerPgNotification bounds.
      - WORKER_PG_QUEUE_CONSUMER_VT_SECONDS=${PG_NOTIFICATION_VT_SECONDS:-300}
      - WORKER_PG_QUEUE_CONSUMER_HEALTH_STALE_SECONDS=${PG_NOTIFICATION_HEALTH_STALE_SECONDS:-360}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config
      - prompt_studio_data:/app/prompt-studio-data

  # Reaper / orchestrator — leader-elected loop. Run exactly ONE instance (it
  # elects a single leader via pg_orchestrator_lock; extra replicas idle as
  # standby). Besides barrier-orphan recovery it runs the PG scheduler tick
  # (fires due pg_owned schedules — Beat replacement); dark until rows are owned.
  worker-pg-reaper:
    image: unstract/worker-unified:${VERSION}
    container_name: unstract-worker-pg-reaper
    restart: unless-stopped
    command: ["pg-queue-reaper"]
    ports:
      - "8097:8086"
    env_file:
      - ../workers/.env
      - ./essentials.env
    depends_on:
      - db
      - redis
    environment:
      - ENVIRONMENT=development
      - APPLICATION_NAME=unstract-worker-pg-reaper
      - WORKER_PG_REAPER_INTERVAL_SECONDS=${WORKER_PG_REAPER_INTERVAL_SECONDS:-5}
      - WORKER_PG_REAPER_HEALTH_PORT=8086
      # Log publisher — see the note on the backend service (UN-4046).
      - LOG_TRANSPORT=redis
      # Retention sweep (cadence-gated inside the leader's tick): drop expired
      # pg_task_result rows + orphaned pg_batch_dedup markers so neither grows
      # unbounded as the gate ramps. The dedup retention below (default 24h) should
      # be kept above the longest possible execution, so an in-flight marker is
      # never swept.
      - WORKER_PG_REAPER_SWEEP_SECONDS=${WORKER_PG_REAPER_SWEEP_SECONDS:-300}
      - WORKER_PG_DEDUP_RETENTION_SECONDS=${WORKER_PG_DEDUP_RETENTION_SECONDS:-86400}
    labels:
      - traefik.enable=false
    volumes:
      - ./workflow_data:/data
      - ${TOOL_REGISTRY_CONFIG_SRC_PATH}:/data/tool_registry_config

volumes:
  prompt_studio_data:
  unstract_data:

networks:
  default:
    # NOTE:
    # Any changes need to be reflected in proxy service too.
    name: unstract-network