Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

5.6 KB
Raw Blame
# To pass to tool-sidecar for Kombu's connection
CELERY_BROKER_BASE_URL="amqp://unstract-rabbitmq:5672//"
CELERY_BROKER_USER=admin
CELERY_BROKER_PASS=password

TOOL_CONTAINER_NETWORK="unstract-network"
TOOL_CONTAINER_LABELS="[]"

PRIVATE_REGISTRY_CREDENTIAL_PATH=
PRIVATE_REGISTRY_USERNAME=
PRIVATE_REGISTRY_URL=

# Log level for runner (Options: INFO, WARNING, ERROR, DEBUG, etc.)
LOG_LEVEL="INFO"

# Flag to decide whether to clean up/ remove the tool container after execution.
# (Default: True)
REMOVE_CONTAINER_ON_EXIT=True

# Client module path of the container engine to be used.
CONTAINER_CLIENT_PATH=unstract.runner.clients.docker_client

# Logs Expiry of 24 hours
LOGS_EXPIRATION_TIME_IN_SECOND=86400

# Feature Flags
FLIPT_SERVICE_AVAILABLE=False
EVALUATION_SERVER_IP=unstract-flipt
EVALUATION_SERVER_PORT=9005
PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python

# File System Configuration for Workflow and API Execution
# Directory Prefixes for storing execution files
WORKFLOW_EXECUTION_DIR_PREFIX="unstract/execution"
# Storage Provider for Workflow Execution
# Valid options: MINIO, S3, etc..
WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'

# For unified notification
REDIS_HOST=unstract-redis
REDIS_PORT=6379
REDIS_USER=default
REDIS_PASSWORD=

# Redis Sentinel mode - set to True for HA deployments (K8s only)
# When True: point REDIS_HOST to Sentinel K8s service DNS, REDIS_PORT to 26379
# REDIS_PASSWORD is reused for Sentinel auth.
REDIS_SENTINEL_MODE=False
REDIS_SENTINEL_MASTER_NAME=mymaster

# Managed / external Redis with TLS (UN-4123). All optional — unset means the
# plaintext connection this file otherwise describes, unchanged.
#
# Two ways to configure it, pick one:
#   1. Discrete vars (what the Helm chart and these samples use). Set REDIS_SSL=true
#      alongside REDIS_HOST/REDIS_PORT. No URL-encoding to get wrong.
#   2. REDIS_URL, where the SCHEME carries TLS and nothing else is needed:
#        REDIS_URL=rediss://<host>:6380/0?ssl_cert_reqs=required
#        REDIS_PASSWORD=<password>
#        REDIS_USER=                        # blank — see below
#        (The password is better kept OUT of the URL. A URL is printed into log
#         lines, error messages and deployment tooling; a password in one travels
#         with it. Credentials in the URL still work and still win, so an existing
#         rediss://:<password>@host URL keeps behaving exactly as before.)
#        (6380 is an EXAMPLE, not a default — the TLS port is provider-specific:
#         Memorystore 6378, ElastiCache 6379, Azure Cache 6380. A wrong port
#         hangs the connection rather than erroring, so read it off the instance.)
#      A URL wins over the discrete vars above. Percent-encode @ / + in the password.
#
# Auth is password-only, as Redis's built-in `default` user — that is what a managed
# AUTH string is. Leave REDIS_USER empty for a managed endpoint; named ACL users are
# not supported platform-wide.
#
# REDIS_SSL_CA_CERTS is only needed when the server's CA is not in the system trust
# store (Memorystore). ElastiCache and Azure Cache chain to public CAs.
#
# CAVEAT — it names a path INSIDE each container, and the runner mounts only the
# shared logs volume into tool containers and sidecars. Nothing in this repo puts
# a CA file there, so unless your images already bake one in at this exact path,
# setting it makes those containers fail: SSLConnection calls
# load_verify_locations() on a path that does not exist. The sidecar then cannot
# build its LogPublisher client and tool logs are lost; in tool containers
# MetricsMixin swallows it and timings disappear. The variable IS forwarded, so a
# baked-in CA works — but a path that only exists on the host does not, and
# against a publicly-trusted endpoint it turns a working connection into a failing
# one. docker/redis-tls/README.md shows how to mount it for local development.
# REDIS_SSL=false
# REDIS_SSL_CERT_REQS=required
# REDIS_SSL_CA_CERTS=
#
# Hostname verification is ON by default (UN-4123): a chain verified against a
# public CA still says nothing about WHICH server answered. Set this to false for
# an endpoint whose certificate does not cover how you address it — an IP, or an
# internal CNAME. Forced off anyway when REDIS_SSL_CERT_REQS=none, and Sentinel
# MASTER connections skip it by default because Sentinel returns an IP that no
# DNS SAN covers and that changes on failover. Forwarded to each tool sidecar,
# so a platform-wide `false` reaches them too — without that they alone would
# keep verifying and fail the handshake while everything else recovered.
# Shown as `false` because that is the only value there is a reason to set:
# `true` is already the default, and setting it EXPLICITLY is not a no-op —
# an explicit value is honoured on Sentinel MASTER connections too, turning
# the compatibility default above back off.
# REDIS_SSL_CHECK_HOSTNAME=false
# REDIS_URL=

# Proactive health check for pooled connections, in seconds (0 disables).
# Managed Redis fails over during maintenance and reaps idle connections — Azure
# Cache closes them at 10 minutes — so a parked connection can be dead before its
# next command. Defaults to 30.
# REDIS_HEALTH_CHECK_INTERVAL=30

# Flask related envs
# Can be 'production' or 'development'
FLASK_ENV=production
FLASK_RUN_HOST=0.0.0.0
FLASK_RUN_PORT=5002
FLASK_APP=unstract.runner:app

# Tool Sidecar
TOOL_SIDECAR_ENABLED=False
TOOL_SIDECAR_IMAGE_NAME="unstract/tool-sidecar"
TOOL_SIDECAR_IMAGE_TAG="0.2.1"
TOOL_EXECUTION_CACHE_TTL_IN_SECOND=86400 # 24 Hours

# File Execution Tracker
FILE_EXECUTION_TRACKER_TTL_IN_SECOND=18000 # 5 hours