LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
# E2E overlay for docker/docker-compose.yaml — override only what the test
# stack needs, so e2e runs against production-like images.
services:
backend:
# CI pins the tag to the SHA built in the same run; `latest` lets a
# contributor run without setting it.
image: unstract/backend:${UNSTRACT_TEST_VERSION:-latest}
environment:
# Workers ask the backend for this, so this is the value that normally
# takes effect for fan-out.
- MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}
# Lets the agent-kv e2e lane's page-cap scenario set a cap smaller than
# its 2-page fixture (default matches AGENT_KV_MAX_PAGES's own default
# in docker/sample.env; the test skips unless this is overridden <=2).
- AGENT_KV_MAX_PAGES=${AGENT_KV_MAX_PAGES:-100}
# Bucket-rooted object-store root for staged inputs/results; the first
# segment is the bucket and must exist. Must be identical to the value
# given to worker-executor-v2 below -- the executor reads back what the
# backend staged and caches OCR under the same root.
- AGENT_KV_STORAGE_DIR_PREFIX=${AGENT_KV_STORAGE_DIR_PREFIX:-unstract/agent_kv}
# Flips on the Agent-KV calc e2e lane (Task 12); unset/empty (the
# default) keeps it off so other lanes are unaffected.
- AGENT_KV_CALCULATIONS_ENABLED=${AGENT_KV_CALCULATIONS_ENABLED:-}
# The webhook SSRF waiver, which the backend needs as well as
# worker-pg-ide-callback below: the https-and-public-host rule is
# applied at SUBMIT as well as at delivery, so without it here the
# webhook e2e scenario is refused with a 400 before a job exists and
# the worker-side waiver never gets a chance to matter. Both services
# read the same variable; unset (the default) keeps both guards on.
- AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=${AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE:-}
platform-service:
image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest}
runner:
image: unstract/runner:${UNSTRACT_TEST_VERSION:-latest}
# Retargeted onto the PG consumers (UN-4046). The Celery worker services these
# used to name — worker-executor-v2, worker-file-processing-v2,
# worker-api-deployment-v2 — are gone from docker/docker-compose.yaml. An
# override for a service that does not exist is NOT a no-op: compose creates it
# with neither image nor build context and rejects the whole project with
# "invalid compose project", so e2e could not bring the stack up at all.
# Execute-path e2e must never reach a real provider.
worker-pg-executor:
environment:
- UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}
# Add the agent-kv executor's queue to the default set (docs/agent-kv-api.md
# §12 item 2) without dropping the queues other e2e lanes' tools dispatch
# to on this same worker -- this is a superset of docker-compose.yaml's
# own default, not a replacement for it.
#
# This MUST be WORKER_PG_QUEUE_CONSUMER_QUEUE, not CELERY_QUEUES_EXECUTOR:
# since UN-4046 made the PG transport unconditional this service runs
# `pg-queue-consumer`, which reads WORKER_PG_QUEUE_CONSUMER_QUEUE.
# CELERY_QUEUES_EXECUTOR is read only by workers/executor/worker.py -- the
# now-disabled Celery executor -- so setting it here drained nothing and
# the agent-kv lane hung in DISPATCHED with no error at the producer.
- WORKER_PG_QUEUE_CONSUMER_QUEUE=${WORKER_PG_QUEUE_CONSUMER_QUEUE:-celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test}
# Agent-KV engine config, read by the cloud `agentic_table` executor plugin
# (not by backend settings) -- docs/agent-kv-api.md §12 item 5 / §13.
# `AGENT_KV_LLM_PROVIDER` is a v1 hard allowlist of `anthropic`|`openai`.
# Model names have no safe default -- Task 13b must supply real ones.
- AGENT_KV_LLM_PROVIDER=${AGENT_KV_LLM_PROVIDER:-anthropic}
- AGENT_KV_LITE_MODEL=${AGENT_KV_LITE_MODEL:-}
- AGENT_KV_ADVANCED_MODEL=${AGENT_KV_ADVANCED_MODEL:-}
- AGENT_KV_LLM_API_KEY=${AGENT_KV_LLM_API_KEY:-}
- AGENT_KV_LLMWHISPERER_API_KEY=${AGENT_KV_LLMWHISPERER_API_KEY:-}
- AGENT_KV_LLMWHISPERER_BASE_URL=${AGENT_KV_LLMWHISPERER_BASE_URL:-https://llmwhisperer-api.us-central.unstract.com/api/v2}
# Same root the backend stages into (see `backend` above) -- the executor
# reads `input_ref` from it and keys its OCR cache under
# `{prefix}/{org_id}/cache/`. Both sides MUST agree.
- AGENT_KV_STORAGE_DIR_PREFIX=${AGENT_KV_STORAGE_DIR_PREFIX:-unstract/agent_kv}
worker-pg-fileproc:
environment:
- UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}
# Only the worker's fallback if it can't reach the backend; kept in step so it
# can't quietly serialise the fan-out test.
worker-pg-orchestrator-api:
environment:
- MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}
# Re-anchored onto worker-pg-ide-callback: UN-4046 replaced the Celery
# worker-ide-callback service with its PG-transport twin.
# Completion-webhook e2e: waive the worker's SSRF guards (https + public
# host) so a receiver on the compose host is reachable. Empty/unset (the
# default) keeps the guards mandatory; the gated webhook scenario skips.
worker-pg-ide-callback:
environment:
- AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=${AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE:-}
# The `ui` Playwright group drives a browser at the frontend's OWN origin, and
# the app derives every API URL from window.location.origin. The production
# nginx config serves only static files plus an SPA fallback, so `/api/v1/*`
# came back as index.html and authenticated specs never got past login. This
# mounts a test-only config that proxies the API prefixes to the backend,
# giving the suite a single origin. See frontend-nginx.test.conf for why the
# Traefik origin the compose stack already provides is not usable here.
frontend:
volumes:
# Relative to the PROJECT directory (docker/, from the first -f), not to
# this file — Compose resolves every path against the project dir, so a
# bare ./ here would silently resolve to docker/ and Docker would create
# an empty directory at the mount target rather than error.
- ../tests/compose/frontend-nginx.test.conf:/etc/nginx/nginx.conf:ro