Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

11.7 KB
Raw Blame
# Path where public and private tools are registered
# with a YAML and JSONs
TOOL_REGISTRY_CONFIG_SRC_PATH="${PWD}/../unstract/tool-registry/tool_registry_config"

# Celery Autoscaling Configuration
# Specify the maximum and minimum number of concurrent workers for each Celery worker.
# Format: <max_workers>,<min_workers>
# Hint: The max value (max_workers) is related to your CPU resources and the level of concurrency you need.
# Always monitor system performance and adjust the max value as needed.
#
# Worker autoscaling (matches hierarchical configuration below)
WORKER_API_DEPLOYMENT_AUTOSCALE=4,1         # API deployment worker autoscale
WORKER_CALLBACK_AUTOSCALE=4,1               # Callback worker autoscale
WORKER_GENERAL_AUTOSCALE=6,2                # General worker autoscale (enhanced)
WORKER_FILE_PROCESSING_AUTOSCALE=8,2    # File processing unified worker autoscale
WORKER_NOTIFICATION_AUTOSCALE=4,1           # Notification worker autoscale
WORKER_LOG_CONSUMER_AUTOSCALE=2,1           # Log consumer worker autoscale
WORKER_SCHEDULER_AUTOSCALE=2,1              # Scheduler worker autoscale

# Worker-specific configurations
API_DEPLOYMENT_WORKER_NAME=api-deployment-worker
API_DEPLOYMENT_HEALTH_PORT=8080
API_DEPLOYMENT_MAX_CONCURRENT_TASKS=5

CALLBACK_WORKER_NAME=callback-worker
CALLBACK_HEALTH_PORT=8083
CALLBACK_MAX_CONCURRENT_TASKS=3

FILE_PROCESSING_WORKER_NAME=file-processing-worker
FILE_PROCESSING_HEALTH_PORT=8082
FILE_PROCESSING_MAX_CONCURRENT_TASKS=4

GENERAL_WORKER_NAME=general-worker
GENERAL_HEALTH_PORT=8081
GENERAL_MAX_CONCURRENT_TASKS=10

# =============================================================================
# HIERARCHICAL CELERY CONFIGURATION SYSTEM
# =============================================================================
#
# This system uses a 3-tier hierarchy for all Celery settings (most specific wins):
# 1. {WORKER_TYPE}_{SETTING_NAME} - Worker-specific override (highest priority)
# 2. CELERY_{SETTING_NAME} - Global override (medium priority)
# 3. Code default - Celery standard default (lowest priority)
#
# Examples:
# - CALLBACK_TASK_TIME_LIMIT=3600 (callback worker only)
# - CELERY_TASK_TIME_LIMIT=300 (all workers)
# - Code provides default if neither is set
#
# Worker types: API_DEPLOYMENT, GENERAL, FILE_PROCESSING, CALLBACK,
#              NOTIFICATION, LOG_CONSUMER, SCHEDULER
# =============================================================================

# Global Celery Configuration (applies to all workers unless overridden)
CELERY_RESULT_CHORD_RETRY_INTERVAL=3        # Global chord unlock retry interval
CELERY_TASK_TIME_LIMIT=7200                 # Global task timeout (2 hours)
CELERY_TASK_SOFT_TIME_LIMIT=6300            # Global soft timeout (1h 45m)
CELERY_PREFETCH_MULTIPLIER=1                # Global prefetch multiplier
CELERY_MAX_TASKS_PER_CHILD=1000             # Global max tasks per child process
CELERY_TASK_ACKS_LATE=true                  # Global acks late setting
CELERY_TASK_DEFAULT_RETRY_DELAY=60          # Global retry delay (1 minute)
CELERY_TASK_MAX_RETRIES=3                   # Global max retries

# Worker-Specific Configuration Examples
# Callback Worker - Chord settings and extended timeouts
CALLBACK_RESULT_CHORD_RETRY_INTERVAL=3      # Callback-specific chord retry interval
CALLBACK_TASK_TIME_LIMIT=7200               # Callback tasks need more time (2 hours)
CALLBACK_TASK_SOFT_TIME_LIMIT=6300          # Callback soft timeout (1h 45m)

# File Processing Worker - Thread pool and optimized settings
FILE_PROCESSING_POOL_TYPE=threads           # Use threads instead of prefork
FILE_PROCESSING_CONCURRENCY=4               # Fixed concurrency for file processing
FILE_PROCESSING_TASK_TIME_LIMIT=10800       # File processing timeout (3 hours)

# API Deployment Worker - Autoscaling and timeout configuration
API_DEPLOYMENT_AUTOSCALE=4,1                # Max 4, min 1 workers
API_DEPLOYMENT_TASK_TIME_LIMIT=3600         # API deployment timeout (1 hour)

# General Worker - Enhanced scaling for high-throughput tasks
GENERAL_AUTOSCALE=6,2                       # Max 6, min 2 workers

# Docker Worker-Specific Concurrency Settings (for docker-compose.yaml)
WORKER_API_DEPLOYMENT_CONCURRENCY=4         # API deployment fixed concurrency
WORKER_FILE_PROCESSING_CONCURRENCY=8        # File processing fixed concurrency
WORKER_NOTIFICATION_CONCURRENCY=4           # Notification worker concurrency
WORKER_LOG_CONSUMER_CONCURRENCY=2           # Log consumer worker concurrency
WORKER_SCHEDULER_CONCURRENCY=2              # Scheduler worker concurrency

# Notification Worker - Optimized for quick message processing
NOTIFICATION_AUTOSCALE=4,1                  # Max 4, min 1 workers
NOTIFICATION_TASK_TIME_LIMIT=120            # Quick timeout for notifications

# Scheduler Worker - Conservative settings for scheduled tasks
SCHEDULER_AUTOSCALE=2,1                     # Max 2, min 1 workers
SCHEDULER_TASK_TIME_LIMIT=1800              # Scheduler timeout (30 minutes)

# Log Consumer Worker - Optimized for log processing
LOG_CONSUMER_AUTOSCALE=2,1                  # Max 2, min 1 workers
LOG_CONSUMER_TASK_TIME_LIMIT=600            # Log processing timeout (10 minutes)

# IDE Callback Worker (worker-ide-callback) — Agent-KV terminal callbacks
# ---------------------------------------------------------------------------
# The ide_callback worker also owns agent_kv_complete/agent_kv_error
# (Agent-KV job finalize callbacks, spec §5.3) on their own queue,
# `agent_kv_callback`, alongside its existing `ide_callback` queue.
# unstract/backend's agent_kv.dispatch dispatches those tasks straight to
# that queue name, so worker-ide-callback must consume both queues or
# Agent-KV jobs will dispatch but never finalize.
#
# NOTE: unlike CALLBACK/FILE_PROCESSING/etc. above, this worker's queue list
# is NOT overridable from this file — there is no CELERY_QUEUES_IDE_CALLBACK
# var wired into workers/run-worker-docker.sh's queue resolution. The queue
# list is a hardcoded default in that script (WORKER_QUEUES["ide_callback"]),
# already set to "ide_callback,agent_kv_callback" there. Nothing to set here;
# if you ever need a different set for this worker, edit that script's
# default directly (or run-worker.sh's equivalent for non-docker dev runs) —
# there's no env-var lever for it today.

# Agent-KV API (backend/backend/settings/base.py) — see docs/agent-kv-api.md
# ---------------------------------------------------------------------------
# Django app settings for the Agent-KV extraction API (unstract/backend's
# `agent_kv` app). Consumed by the Django backend process, not by
# docker-compose itself — mirror these into backend/.env (backend/sample.env)
# for a real deployment; listed here per the Agent-KV env-sample convention.
AGENT_KV_PATH_PREFIX=api/v1/agent-kv                # Public URL prefix; shares the /api/v1 ingress rule
AGENT_KV_MAX_FILE_SIZE_MB=50                        # Submit-time file size cap (MB)
AGENT_KV_MAX_PAGES=100                              # Pre-OCR page cap for PDFs/images
AGENT_KV_MAX_CALCULATIONS_BYTES=20000               # Byte cap on the optional `calculations` field
AGENT_KV_MAX_SCHEMA_BYTES=262144                    # Byte cap on the raw `keys` JSON schema (256 KiB)
AGENT_KV_RESULT_TTL_DAYS=7                          # Result/input retention window, stamped at submit time
AGENT_KV_MAX_TIMEOUT_SECONDS=300                    # Upper bound on the submit `timeout` (sync-wait) field
AGENT_KV_CONCURRENT_LIMIT=5                         # Per-org concurrent in-flight job cap
AGENT_KV_KEY_RATE_LIMIT_PER_MINUTE=60               # Per-key request rate limit (submit + validate)
AGENT_KV_LIMITER_FAIL_OPEN=false                    # true = both limiters above allow the request when Redis is down (default: refuse with 429)
AGENT_KV_SWEEP_GRACE_SECONDS=3600                   # Age before a never-dispatched PENDING job is swept
AGENT_KV_STUCK_JOB_GRACE_SECONDS=21600              # Age (from dispatch) before a stuck DISPATCHED/RUNNING job is force-failed (6h)
AGENT_KV_CALCULATIONS_ENABLED=false                 # Gates the submit `calculations` field; engine can't run it yet
AGENT_KV_STRUCTURED_OUTPUT_ENABLED=false            # Gates the submit `structured_output` field; engine can't run it yet

# Bucket-rooted object-store root for staged inputs/results
# ({AGENT_KV_STORAGE_DIR_PREFIX}/{org_id}/{job_id}/…), like WORKFLOW_EXECUTION_DIR_PREFIX.
# The FIRST segment is the BUCKET (s3fs/gcsfs treat it so) and must already exist —
# the MinIO dev bootstrap creates `unstract`. MUST match the cloud executor's
# AGENT_KV_STORAGE_DIR_PREFIX: the executor keys its OCR cache under the same root.
AGENT_KV_STORAGE_DIR_PREFIX="unstract/agent_kv"

# Object storage credentials for those files — same shape as
# WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS; required, no default.
AGENT_KV_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'

# Worker Circuit Breaker Settings
CIRCUIT_BREAKER_FAILURE_THRESHOLD=5
CIRCUIT_BREAKER_RECOVERY_TIMEOUT=60

# Worker Health Check Settings
HEALTH_CHECK_INTERVAL=30
HEALTH_CHECK_TIMEOUT=10
ENABLE_METRICS=true

# =============================================================================
# PG-Queue Services (Postgres-backed transport) — gated, opt-in
# =============================================================================
# The PG consumer/reaper services live behind the `pg-queue` compose profile and
# are OFF by default. Bring them up with:
#   docker compose --profile pg-queue up -d
# They consume from Postgres (not the broker); their worker-type/queue identity
# is set per service in docker-compose.yaml, so nothing here is required to run
# them. The values below are documented for reference / overrides only.
#
# WORKER_BARRIER_BACKEND        - fan-in barrier substrate; the PG services set
#                                 this to `pg` (default elsewhere: chord/redis).
# WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE - which worker's tasks the consumer loads.
# WORKER_PG_QUEUE_CONSUMER_QUEUE       - comma-separated queues the consumer polls.
# WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT - opt-in consumer liveness port (unset = off).
# PG_RESULT_SIGNAL_BACKEND      - executor-RPC result delivery: `poll` (default, the
#                                 DB backoff poll) or `redis` (RPUSH/BLPOP wake-up
#                                 signal). The result always stays in Postgres; in
#                                 `redis` mode only the reply key rides Redis, which
#                                 removes the per-wait DB poll so DB load scales with
#                                 throughput, not concurrency. Falls back to the poll
#                                 automatically if Redis is unavailable.
# WORKER_PG_REAPER_HEALTH_PORT         - opt-in reaper liveness port (unset = off).
WORKER_PG_REAPER_INTERVAL_SECONDS=5   # Reaper sweep interval (seconds)
#
# worker-pg-metrics (UN-3796) — drains the dashboard_metrics.* periodics the PG
# scheduler fires, replacing Celery Beat + worker-metrics. Its knobs are sized for a
# task that runs for MINUTES, unlike every other consumer here:
#
# WORKER_PG_METRICS_VT_SECONDS          - claim/drain bound (default 900).
# WORKER_PG_METRICS_HEALTH_STALE_SECONDS- liveness bound (default 960). The heartbeat
#                                 freezes while a task runs, so this is effectively an
#                                 upper bound on ONE task's wall clock — it must stay
#                                 above the vt, which must stay above the task itself.
#                                 above the vt, which must stay above the task itself.
#
# NOTE: since UN-4046 these services ARE the execution path — there is no flag
# and no Celery fallback. Every dispatch goes to PG, so a queue whose consumer is
# not running accepts work and never executes it, with no error at the producer.