Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

6.4 KB
Raw Blame
# E2E overlay for docker/docker-compose.yaml — override only what the test
# stack needs, so e2e runs against production-like images.

services:
  backend:
    # CI pins the tag to the SHA built in the same run; `latest` lets a
    # contributor run without setting it.
    image: unstract/backend:${UNSTRACT_TEST_VERSION:-latest}
    environment:
      # Workers ask the backend for this, so this is the value that normally
      # takes effect for fan-out.
      - MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}
      # Lets the agent-kv e2e lane's page-cap scenario set a cap smaller than
      # its 2-page fixture (default matches AGENT_KV_MAX_PAGES's own default
      # in docker/sample.env; the test skips unless this is overridden <=2).
      - AGENT_KV_MAX_PAGES=${AGENT_KV_MAX_PAGES:-100}
      # Bucket-rooted object-store root for staged inputs/results; the first
      # segment is the bucket and must exist. Must be identical to the value
      # given to worker-executor-v2 below -- the executor reads back what the
      # backend staged and caches OCR under the same root.
      - AGENT_KV_STORAGE_DIR_PREFIX=${AGENT_KV_STORAGE_DIR_PREFIX:-unstract/agent_kv}
      # Flips on the Agent-KV calc e2e lane (Task 12); unset/empty (the
      # default) keeps it off so other lanes are unaffected.
      - AGENT_KV_CALCULATIONS_ENABLED=${AGENT_KV_CALCULATIONS_ENABLED:-}
      # The webhook SSRF waiver, which the backend needs as well as
      # worker-pg-ide-callback below: the https-and-public-host rule is
      # applied at SUBMIT as well as at delivery, so without it here the
      # webhook e2e scenario is refused with a 400 before a job exists and
      # the worker-side waiver never gets a chance to matter. Both services
      # read the same variable; unset (the default) keeps both guards on.
      - AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=${AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE:-}

  platform-service:
    image: unstract/platform-service:${UNSTRACT_TEST_VERSION:-latest}

  runner:
    image: unstract/runner:${UNSTRACT_TEST_VERSION:-latest}

  # Retargeted onto the PG consumers (UN-4046). The Celery worker services these
  # used to name — worker-executor-v2, worker-file-processing-v2,
  # worker-api-deployment-v2 — are gone from docker/docker-compose.yaml. An
  # override for a service that does not exist is NOT a no-op: compose creates it
  # with neither image nor build context and rejects the whole project with
  # "invalid compose project", so e2e could not bring the stack up at all.

  # Execute-path e2e must never reach a real provider.
  worker-pg-executor:
    environment:
      - UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}
      # Add the agent-kv executor's queue to the default set (docs/agent-kv-api.md
      # §12 item 2) without dropping the queues other e2e lanes' tools dispatch
      # to on this same worker -- this is a superset of docker-compose.yaml's
      # own default, not a replacement for it.
      #
      # This MUST be WORKER_PG_QUEUE_CONSUMER_QUEUE, not CELERY_QUEUES_EXECUTOR:
      # since UN-4046 made the PG transport unconditional this service runs
      # `pg-queue-consumer`, which reads WORKER_PG_QUEUE_CONSUMER_QUEUE.
      # CELERY_QUEUES_EXECUTOR is read only by workers/executor/worker.py -- the
      # now-disabled Celery executor -- so setting it here drained nothing and
      # the agent-kv lane hung in DISPATCHED with no error at the producer.
      - WORKER_PG_QUEUE_CONSUMER_QUEUE=${WORKER_PG_QUEUE_CONSUMER_QUEUE:-celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test}
      # Agent-KV engine config, read by the cloud `agentic_table` executor plugin
      # (not by backend settings) -- docs/agent-kv-api.md §12 item 5 / §13.
      # `AGENT_KV_LLM_PROVIDER` is a v1 hard allowlist of `anthropic`|`openai`.
      # Model names have no safe default -- Task 13b must supply real ones.
      - AGENT_KV_LLM_PROVIDER=${AGENT_KV_LLM_PROVIDER:-anthropic}
      - AGENT_KV_LITE_MODEL=${AGENT_KV_LITE_MODEL:-}
      - AGENT_KV_ADVANCED_MODEL=${AGENT_KV_ADVANCED_MODEL:-}
      - AGENT_KV_LLM_API_KEY=${AGENT_KV_LLM_API_KEY:-}
      - AGENT_KV_LLMWHISPERER_API_KEY=${AGENT_KV_LLMWHISPERER_API_KEY:-}
      - AGENT_KV_LLMWHISPERER_BASE_URL=${AGENT_KV_LLMWHISPERER_BASE_URL:-https://llmwhisperer-api.us-central.unstract.com/api/v2}
      # Same root the backend stages into (see `backend` above) -- the executor
      # reads `input_ref` from it and keys its OCR cache under
      # `{prefix}/{org_id}/cache/`. Both sides MUST agree.
      - AGENT_KV_STORAGE_DIR_PREFIX=${AGENT_KV_STORAGE_DIR_PREFIX:-unstract/agent_kv}

  worker-pg-fileproc:
    environment:
      - UNSTRACT_LLM_MOCK_RESPONSE=${UNSTRACT_LLM_MOCK_RESPONSE:-}

  # Only the worker's fallback if it can't reach the backend; kept in step so it
  # can't quietly serialise the fan-out test.
  worker-pg-orchestrator-api:
    environment:
      - MAX_PARALLEL_FILE_BATCHES=${MAX_PARALLEL_FILE_BATCHES:-3}

  # Re-anchored onto worker-pg-ide-callback: UN-4046 replaced the Celery
  # worker-ide-callback service with its PG-transport twin.
  # Completion-webhook e2e: waive the worker's SSRF guards (https + public
  # host) so a receiver on the compose host is reachable. Empty/unset (the
  # default) keeps the guards mandatory; the gated webhook scenario skips.
  worker-pg-ide-callback:
    environment:
      - AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE=${AGENT_KV_WEBHOOK_INSECURE_ALLOW_HTTP_PRIVATE:-}

  # The `ui` Playwright group drives a browser at the frontend's OWN origin, and
  # the app derives every API URL from window.location.origin. The production
  # nginx config serves only static files plus an SPA fallback, so `/api/v1/*`
  # came back as index.html and authenticated specs never got past login. This
  # mounts a test-only config that proxies the API prefixes to the backend,
  # giving the suite a single origin. See frontend-nginx.test.conf for why the
  # Traefik origin the compose stack already provides is not usable here.
  frontend:
    volumes:
      # Relative to the PROJECT directory (docker/, from the first -f), not to
      # this file — Compose resolves every path against the project dir, so a
      # bare ./ here would silently resolve to docker/ and Docker would create
      # an empty directory at the mount target rather than error.
      - ../tests/compose/frontend-nginx.test.conf:/etc/nginx/nginx.conf:ro