Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

11.7 KB
Raw Blame
# Unstract test rig — group definitions.
#
# Each group is the unit of selection. A group's `paths` are passed to pytest
# (or another runner). `depends_on` is expanded transitively before execution.
#
# Add new groups by appending to this file. Validate with:
#   python -m tests.rig validate
#   python -m tests.rig list-groups
#   python -m tests.rig expand <group>
version: 1

defaults:
  timeout_seconds: 600
  parallel: true
  runner: pytest

  # Shared so migrations run under the same settings the tests read.
  backend_test_env: &backend_test_env
    DJANGO_SETTINGS_MODULE: backend.settings.test
    # Tenancy is row-level, not schema-per-tenant.
    DB_SCHEMA: public
    # Resolved at import with no default.
    DJANGO_SECRET_KEY: test-secret-key-not-for-production
    # All-zero Fernet key: valid format, obvious placeholder.
    ENCRYPTION_KEY: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
    CELERY_BROKER_BASE_URL: redis://localhost:6379
    CELERY_BROKER_USER: guest
    CELERY_BROKER_PASS: guest
    INDEXING_FLAG_TTL: "3600"
    ENABLE_LOG_HISTORY: "False"
    STRUCTURE_TOOL_IMAGE_URL: docker:test
    STRUCTURE_TOOL_IMAGE_NAME: test-structure-tool
    STRUCTURE_TOOL_IMAGE_TAG: test
    SYSTEM_ADMIN_USERNAME: admin
    SYSTEM_ADMIN_PASSWORD: admin
    SYSTEM_ADMIN_EMAIL: [email protected]
    # Only used to build paths at init.
    WORKFLOW_EXECUTION_DIR_PREFIX: /tmp/unstract-workflow-exec

  # Provisioned DB gets its schema from the real migration, not fixture DDL,
  # since the tables are Django-managed models owned by the backend.
  postgres_migrate:
    workdir: backend
    apps: [pg_queue]
    env: *backend_test_env

groups:
  # ── Unit tier: pure pytest, no external services ───────────────────────────
  frontend:
    # Vitest + React Testing Library over jsdom — the antd→shadcn shims and the
    # components built on them. Node, not Python: no uv_sync_group, and the
    # runner points vitest's own JUnit reporter at the group's junit.xml, so
    # results come through per-test rather than as one synthetic row.
    #
    # Required: a red test gates the unit tier. It needs `node_modules`, so
    # ci-test.yaml installs frontend deps on the unit leg; with them missing,
    # `npx --no-install vitest` fails and turns the group red. A host with no
    # npx at all (cloud's self-hosted runner) still skips via the runner's
    # exit 5, which folds green for non-pytest groups.
    tier: unit
    runner: vitest
    workdir: frontend
    paths: [src]
    timeout_seconds: 900

  unit-rig:
    # Rig self-tests. Run from repo root so the `tests.rig` import resolves.
    tier: unit
    workdir: .
    paths: [tests/rig/tests]
    coverage_source: tests/rig

  unit-sdk1:
    tier: unit
    workdir: unstract/sdk1
    paths: [tests]
    markers: "not slow and not integration"
    uv_sync_group: test
    coverage_source: src/unstract/sdk1

  unit-platform-service:
    tier: unit
    workdir: platform-service
    paths: [tests]
    uv_sync_group: test
    coverage_source: src
    optional: true   # service has only a couple of tests today

  unit-workers:
    tier: unit
    workdir: workers
    paths: [tests, shared/tests, shared/infrastructure/config/tests,
            ide_callback/tests]
    # Negative filter: worker unit tests aren't tagged `unit` (~1100 of them),
    # so select by exclusion. `conftest.py` marks real-Postgres tests
    # `integration` (routed to the `integration-workers` group); `slow` tests are
    # also excluded from this fast lane. (`markers: "unit"` here previously
    # matched *zero* tests — the suite silently never ran in CI.)
    markers: "not integration and not slow"
    uv_sync_group: test
    coverage_source: shared

  unit-backend:
    tier: unit
    workdir: backend
    # Marker-selected, not a hand-kept file list: live-infra tests carry
    # `integration` and run in integration-backend.
    paths: ["."]
    markers: "not integration"
    uv_sync_group: test
    env: *backend_test_env
    coverage_source: .

  unit-connectors:
    tier: unit
    workdir: unstract/connectors
    paths: [tests]
    # Pure connector tests only. Credential / live-infra tests are marked
    # `@pytest.mark.integration` and run in `integration-connectors`.
    markers: "not integration"
    uv_sync_group: test
    coverage_source: src

  unit-core:
    tier: unit
    workdir: unstract/core
    paths: [tests]
    # No `test` uv group in unstract/core today; rig still injects pytest plugins.
    install_editable: true
    coverage_source: src

  # Both of these had tests but NO group, so nothing collected them. For
  # agent-kv-schema that mattered specifically: test_compile.py is the
  # regression net for the schema depth/ReDoS caps, so the caps were asserted
  # only on a developer's machine. A PR reporting "165 passed" for this package
  # was reporting a local figure, not a CI one.
  unit-agent-kv-schema:
    tier: unit
    workdir: unstract/agent-kv-schema
    paths: [tests]
    install_editable: true
    coverage_source: src

  unit-filesystem:
    tier: unit
    workdir: unstract/filesystem
    paths: [tests]
    install_editable: true
    coverage_source: src

  # runner/tests was registered in NO group, so it was collected by nothing. Its
  # only suite guards that LOG_TRANSPORT and LOG_STREAM_QUEUE_NAME reach the tool
  # sidecar's hand-picked env allowlist — a list that drops anything not named in
  # it, so a missing entry means the sidecar silently logs to the wrong transport.
  # Nothing else covers that, and it went in alongside the Redis log transport.
  # install_editable because the suite imports `unstract.runner`, which resolves
  # only from this package's own src layout (packages = ["src/unstract"]).
  unit-runner:
    tier: unit
    workdir: runner
    paths: [tests]
    uv_sync_group: test
    install_editable: true
    coverage_source: src

  # Guards that the sidecar's startup check asks for the Celery broker only on the
  # Celery log transport. No `test` uv group in tool-sidecar; rig injects pytest.
  unit-tool-sidecar:
    tier: unit
    workdir: tool-sidecar
    paths: [tests]
    install_editable: true
    coverage_source: src

  # ── Integration tier: needs infra but not full platform ────────────────────
  integration-backend:
    tier: integration
    workdir: backend
    # Complement of unit-backend: conftest auto-marks DB-bound tests
    # `integration`, so new ones join here with no edit. The rig provisions
    # requires_services and injects their env; cred-gated tests skip without.
    paths: ["."]
    markers: "integration"
    uv_sync_group: test
    env: *backend_test_env
    requires_services: [postgres, redis, minio]
    coverage_source: .

  integration-workers:
    tier: integration
    workdir: workers
    paths: [tests, shared/tests]
    # Real-Postgres tests, auto-marked `integration` by conftest and connecting
    # over TEST_DB_*, which the rig points at the provisioned container.
    markers: "integration"
    uv_sync_group: test
    coverage_source: shared
    requires_services: [postgres]
    # Optional until REQUIRE_PG_TESTS=1 turns the fixtures' graceful skips into
    # hard failures, so the lane can't pass having run nothing.
    optional: true

  integration-connectors:
    tier: integration
    workdir: unstract/connectors
    paths: [tests]
    markers: "integration"
    # Most need real third-party creds and skip when absent; MinIO runs
    # because the rig provisions it and injects the creds.
    requires_services: [minio]
    uv_sync_group: test
    coverage_source: src

  integration-workflow-execution:
    tier: integration
    paths: [tests/integration/workflow_execution]
    requires_services: [postgres, redis, minio]
    depends_on: [unit-sdk1, unit-workers]
    optional: true   # placeholder until tests are authored

  # ── E2E tier: real HTTP against running platform ───────────────────────────
  # Platform groups must depend (transitively) on e2e-smoke; the rig enforces
  # this so smoke gates and runs first. Nothing ships bypassing it.
  e2e-smoke:
    tier: e2e
    paths: [tests/e2e/smoke]
    requires_platform: true
    critical: true
    timeout_seconds: 1200
    depends_on: [unit-sdk1, unit-workers]

  # Not optional: login is a gated critical path; red fails the build.
  e2e-login:
    tier: e2e
    paths: [tests/e2e/auth]
    requires_platform: true
    critical: true
    depends_on: [e2e-smoke]

  # Owner-side co-owner HTTP contract (endpoints reachable, CSRF, org-scoped,
  # validation + last-owner guard). Gating — CI-verified green before dropping
  # the optional flag (the e2e stack can't run against a dev auth0 backend).
  e2e-coowners:
    tier: e2e
    paths: [tests/e2e/coowners]
    requires_platform: true
    critical: true
    depends_on: [e2e-smoke]

  e2e-workflow:
    tier: e2e
    paths: [tests/e2e/workflows]
    requires_platform: true
    critical: true
    depends_on: [e2e-smoke]

  e2e-api-deployment:
    tier: e2e
    paths: [tests/e2e/api_deployment]
    requires_platform: true
    critical: true
    depends_on: [e2e-smoke]

  e2e-prompt-studio:
    tier: e2e
    paths: [tests/e2e/prompt_studio]
    requires_platform: true
    critical: true
    depends_on: [e2e-smoke]

  e2e-etl:
    tier: e2e
    paths: [tests/e2e/etl]
    requires_platform: true
    critical: true
    depends_on: [e2e-smoke]

  e2e-hurl:
    tier: e2e
    runner: hurl
    paths: [tests/e2e/hurl]
    requires_platform: true
    depends_on: [e2e-smoke]
    optional: true

  # Agent-KV product API (/agent-kv/...). Needs the cloud `agentic_kv`
  # executor plugin installed (docs/agent-kv-api.md §11) -- an OSS-only image
  # 501s every submit, so this is optional: an OSS-only CI run must not go
  # red just because the plugin isn't there. The module itself additionally
  # self-gates on AGENT_KV_E2E=1 so it never runs by accident even when the
  # platform happens to be up (see tests/e2e/agent_kv/test_agent_kv_e2e.py).
  # Longer budget than the other platform groups: the happy-path and delete
  # scenarios poll a real extraction job through to COMPLETED.
  e2e-agent-kv:
    tier: e2e
    paths: [tests/e2e/agent_kv]
    requires_platform: true
    depends_on: [e2e-smoke]
    optional: true
    timeout_seconds: 1200
    # Serial, not xdist-parallel: the concurrency scenario deliberately
    # saturates the org's AgentKVConcurrencyLimiter slots, which
    # JobCancelView never releases (only a finalize callback does, by
    # design -- see the scenario's own docstring). Running sibling
    # submit-based tests on other xdist workers at the same moment would
    # let that saturation 429 them too.
    parallel: false

  ui:
    # Playwright browser e2e over the real UI. Scaffolding only for now — the
    # spec directory holds a single smoke test; extend it here.
    #
    # This is the tier that the 348 vitest tests structurally cannot cover:
    # jsdom computes no layout, so overflow, positioning and visual state are
    # unobservable there. Every UI defect found during the shadcn migration was
    # spotted by a human in a browser, not by the unit suite.
    #
    # `requires_platform` so the rig boots the stack and exports the endpoints:
    # the Playwright config takes its baseURL from UNSTRACT_FRONTEND_URL, and
    # fixtures.js drives the API through UNSTRACT_BACKEND_URL.
    # `optional` until the suite is real: no browsers installed (or no
    # node_modules) must skip, not gate merges.
    tier: e2e
    runner: playwright
    # Self-contained: specs, playwright.config.js and @playwright/test all live
    # in tests/e2e/ui, colocated with the other e2e suites.
    workdir: tests/e2e/ui
    paths: ["."]
    requires_platform: true
    depends_on: [e2e-smoke]
    timeout_seconds: 1800
    optional: true