Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

13.5 KB
Raw Blame
"""E2E fixtures: a session-scoped ``platform`` fixture that yields URLs/creds.

The rig brings the platform up at the *rig* level (once per ``run`` invocation,
not per pytest session) and propagates URLs into pytest via env vars. This
conftest reads those env vars; if they're missing, e2e tests are skipped with
a clear message rather than spuriously failing.
"""

from __future__ import annotations

import importlib
import os
import time
import uuid
from collections.abc import Callable
from dataclasses import dataclass

import pytest
import requests

from tests.rig.runtime import (
    LLM_MOCK_RESPONSE_ENV,
    PlatformEndpoints,
)

# A downstream repo (e.g. cloud with an Auth0 login) can supply its own login by
# pointing this at a "module.func" that takes PlatformEndpoints and returns a
# logged-in requests.Session. Unset → the OSS form login below.
LOGIN_PROVIDER_ENV = "UNSTRACT_E2E_LOGIN_PROVIDER"

# Mirrors unstract.core.data_models.ExecutionStatus.terminal_statuses(); kept a
# local literal because the e2e venv carries only pytest/requests/minio, and
# these are the JSON status strings the HTTP API returns anyway.
TERMINAL_EXECUTION_STATUSES = {"COMPLETED", "ERROR", "STOPPED"}

class CsrfSession(requests.Session):
    """Stamps X-CSRFToken from its own cookies on unsafe methods.

    Centralises what a handful of per-file helpers each re-implemented (and had
    already drifted on — some merged caller headers, some dropped them).
    """

    _UNSAFE = frozenset({"POST", "PUT", "PATCH", "DELETE"})

    def request(self, method: str, url: str, **kwargs: object) -> requests.Response:
        if method.upper() in self._UNSAFE:
            headers = dict(kwargs.get("headers") or {})
            headers.setdefault("X-CSRFToken", self.cookies.get("csrftoken", ""))
            kwargs["headers"] = headers
        return super().request(method, url, **kwargs)


# Adapter registry ids from unstract.sdk1, chosen so no real provider is called:
# the vectordb/x2text NoOp rows return canned output and the LLM is mocked. The
# embedding adapter is real but never hit — chunk_size=0 keeps indexing off the
# path. Fake creds persist because adapter create does not validate connectivity.
_LLM_ADAPTER = "openai|502ecf49-e47c-445c-9907-6d4b90c5cd17"
_EMBED_ADAPTER = "openai|717a0b0e-3bbc-41dc-9f0c-5689437a1151"
_VECTORDB_ADAPTER = "noOpVectorDb|ca4d6056-4971-4bc8-97e3-9e36290b5bc0"
_X2TEXT_ADAPTER = "noOpX2text|mp66d1op-7100-d340-9101-846fc7115676"


@pytest.fixture(scope="session")
def platform() -> PlatformEndpoints:
    if not os.environ.get("UNSTRACT_BACKEND_URL"):
        pytest.skip(
            "platform URLs not set in env; run via "
            "`python -m tests.rig run --tier e2e ...` or export "
            "UNSTRACT_BACKEND_URL (etc.) yourself."
        )
    return PlatformEndpoints.from_env()


def _load_login_provider() -> Callable[[PlatformEndpoints], requests.Session]:
    """Resolve the login callable from LOGIN_PROVIDER_ENV, else the OSS one."""
    dotted = os.environ.get(LOGIN_PROVIDER_ENV, "").strip()
    if not dotted:
        return _oss_form_login
    module_name, _, func_name = dotted.rpartition(".")
    if not module_name or not func_name:
        raise ValueError(
            f"{LOGIN_PROVIDER_ENV} must be a 'module.func' path, got {dotted!r}"
        )
    provider = getattr(importlib.import_module(module_name), func_name)
    if not callable(provider):
        raise ValueError(
            f"{LOGIN_PROVIDER_ENV}={dotted!r} does not resolve to a callable"
        )
    return provider


@pytest.fixture(scope="session")
def authed_session(platform: PlatformEndpoints) -> requests.Session:
    """A logged-in session with the active organization set.

    Session-scoped: logged in once, reused across tests. Tests that exercise
    login itself should build their own session instead. The login flow is
    pluggable via LOGIN_PROVIDER_ENV so a downstream repo can swap in its own
    (e.g. Auth0) while every non-auth test reuses this one seam unchanged.
    """
    return _load_login_provider()(platform)


def _oss_form_login(platform: PlatformEndpoints) -> requests.Session:
    """The OSS mock-login flow: form POST to /api/v1/login (302 + sessionid),
    then the org handshake (GET /organization seeds the csrftoken cookie, POST
    /organization/{id}/set with X-CSRFToken) so org-scoped endpoints work.
    """
    base = platform.backend_url.rstrip("/")
    session = CsrfSession()

    resp = session.post(
        f"{base}/api/v1/login",
        data={"username": platform.admin_user, "password": platform.admin_password},
        allow_redirects=False,
        timeout=10,
    )
    assert resp.status_code == 302, (
        f"login failed: expected 302, got {resp.status_code} "
        "(200 with HTML means bad credentials)"
    )

    # GET /organization sets the csrftoken cookie and returns the org id, which
    # the following POST needs both as a cookie and echoed in the CSRF header.
    orgs = session.get(f"{base}/api/v1/organization", timeout=10)
    orgs.raise_for_status()
    org_list = orgs.json().get("organizations", [])
    assert org_list, (
        f"no organizations returned by /api/v1/organization — is seed data "
        f"loaded? response: {orgs.text}"
    )
    org_id = org_list[0]["id"]
    resp = session.post(
        f"{base}/api/v1/organization/{org_id}/set",
        headers={"X-CSRFToken": session.cookies.get("csrftoken", "")},
        timeout=10,
    )
    resp.raise_for_status()
    return session


@pytest.fixture(scope="session")
def llm_mock_response() -> str:
    """The exact string the workers were told to return for every completion.

    Skips rather than guesses when unset: there is no deterministic answer to
    assert unless the workers got the same value.
    """
    value = os.environ.get(LLM_MOCK_RESPONSE_ENV)
    if not value:
        # Under a rig run the value is mandatory; skipping green would silently
        # drop the whole execute path. Outside the rig, skip is still right.
        if os.environ.get("UNSTRACT_RIG_SESSION_ID"):
            pytest.fail(
                f"{LLM_MOCK_RESPONSE_ENV} unset under a rig run — the rig must set "
                "it before booting the platform."
            )
        pytest.skip(
            f"{LLM_MOCK_RESPONSE_ENV} not set — execute-path e2e needs the LLM mock."
        )
    return value


@dataclass(frozen=True)
class ProvisionedWorkflow:
    """Handles for a hermetic, execute-ready workflow (one Prompt Studio tool)."""

    session: requests.Session
    base: str  # backend root, e.g. http://localhost:8000
    prefix: str  # tenant-scoped API root: {base}/api/v1/unstract/{org_id}
    org_id: str
    workflow_id: str
    tool_id: str
    prompt_registry_id: str
    profile_id: str
    prompt_id: str
    prompt_key: str


def _org_id(session: requests.Session, base: str) -> str:
    orgs = session.get(f"{base}/api/v1/organization", timeout=10)
    orgs.raise_for_status()
    return orgs.json()["organizations"][0]["id"]


def _post(session: requests.Session, url: str, **kw: object) -> requests.Response:
    return session.post(url, timeout=60, **kw)  # CsrfSession stamps the header


def _patch(session: requests.Session, url: str, **kw: object) -> requests.Response:
    return session.patch(url, timeout=60, **kw)


def wait_for_execution(
    session: requests.Session,
    prefix: str,
    execution_id: str,
    *,
    timeout: float = 300.0,
) -> dict:
    """Poll an execution until terminal, failing loudly on timeout.

    A timed-out poll fails with the last-seen body so a slow worker reads
    differently from a genuine terminal failure.
    """
    deadline = time.monotonic() + timeout
    execution: dict = {}
    while time.monotonic() < deadline:
        resp = session.get(f"{prefix}/execution/{execution_id}/", timeout=30)
        resp.raise_for_status()
        execution = resp.json()
        if execution.get("status") in TERMINAL_EXECUTION_STATUSES:
            return execution
        time.sleep(2)
    pytest.fail(f"execution not terminal within {timeout}s: {execution}")


@pytest.fixture(scope="session")
def provisioned_workflow(
    platform: PlatformEndpoints, authed_session: requests.Session
) -> ProvisionedWorkflow:
    """Stand up an API workflow backed by a single Prompt Studio tool.

    Provisioned over HTTP rather than via the ORM so setup exercises the same
    surface as the app. Executes hermetically: the LLM is mocked, and
    chunk_size=0 keeps embedding and vectordb out of the path.
    """
    s = authed_session
    base = platform.backend_url.rstrip("/")
    org_id = _org_id(s, base)
    prefix = f"{base}/api/v1/unstract/{org_id}"
    sfx = uuid.uuid4().hex[:8]  # adapter/tool names are unique per org

    def create_adapter(adapter_id: str, adapter_type: str, metadata: dict) -> str:
        name = f"e2e-{adapter_type.lower()}-{sfx}"
        resp = _post(
            s,
            f"{prefix}/adapter/",
            json={
                "adapter_id": adapter_id,
                "adapter_name": name,
                "adapter_type": adapter_type,
                "adapter_metadata": {"adapter_name": name, **metadata},
            },
        )
        assert resp.status_code == 201, f"adapter {adapter_type}: {resp.text}"
        return resp.json()["id"]

    # api_base is required by the openai adapter's JSON schema, so adapter
    # creation rejects the row without it even though the completion is mocked.
    llm_id = create_adapter(
        _LLM_ADAPTER,
        "LLM",
        {
            "api_key": "sk-test",
            "model": "gpt-4o",
            "api_base": "https://api.openai.com/v1",
        },
    )
    embed_id = create_adapter(
        _EMBED_ADAPTER,
        "EMBEDDING",
        {"api_key": "sk-test", "model": "text-embedding-3-small"},
    )
    vdb_id = create_adapter(_VECTORDB_ADAPTER, "VECTOR_DB", {"wait_time": 0})
    x2t_id = create_adapter(_X2TEXT_ADAPTER, "X2TEXT", {"wait_time": 0})

    resp = _post(
        s,
        f"{prefix}/prompt-studio/",
        json={"tool_name": f"e2e-{sfx}", "description": "e2e execute", "author": "e2e"},
    )
    assert resp.status_code == 201, f"prompt-studio project: {resp.text}"
    tool_id = resp.json()["tool_id"]

    # Patch the auto-created default profile; a second one would break export.
    resp = s.get(f"{prefix}/prompt-studio/prompt-studio-profile/{tool_id}/", timeout=30)
    resp.raise_for_status()
    profile_id = next(p["profile_id"] for p in resp.json() if p.get("is_default"))
    resp = _patch(
        s,
        f"{prefix}/prompt-studio/profile-manager/{profile_id}/",
        json={
            "llm": llm_id,
            "embedding_model": embed_id,
            "vector_store": vdb_id,
            "x2text": x2t_id,
            "chunk_size": 0,
            "chunk_overlap": 0,
        },
    )
    assert resp.status_code == 200, f"profile patch: {resp.text}"

    prompt_key = "answer"
    resp = _post(
        s,
        f"{prefix}/prompt-studio/prompt-studio-prompt/{tool_id}/",
        json={
            "tool_id": tool_id,
            "prompt_key": prompt_key,
            "prompt": "What is this document about?",
            "prompt_type": "PROMPT",
            # text keeps the answer the raw completion; the structured types are
            # re-serialised on the way out and would not match the mock verbatim.
            "enforce_type": "text",
            "sequence_number": 1,
            "active": True,
            "profile_manager": profile_id,
        },
    )
    assert resp.status_code == 201, f"add prompt: {resp.text}"
    prompt_id = resp.json()["prompt_id"]

    resp = _post(
        s,
        f"{prefix}/prompt-studio/export/{tool_id}",
        json={"force_export": True, "is_shared_with_org": True},
    )
    assert resp.status_code == 200, f"export: {resp.text}"
    # The filter arg is mandatory: without one the view returns no queryset.
    resp = s.get(
        f"{prefix}/prompt-studio/registry/", params={"custom_tool": tool_id}, timeout=30
    )
    resp.raise_for_status()
    regs = resp.json()
    reg_list = regs if isinstance(regs, list) else regs.get("results", [])
    assert reg_list, f"registry empty for tool {tool_id}"
    prompt_registry_id = reg_list[0]["prompt_registry_id"]

    resp = _post(s, f"{prefix}/workflow/", json={"workflow_name": f"e2e-wf-{sfx}"})
    assert resp.status_code == 201, f"create workflow: {resp.text}"
    workflow_id = resp.json()["id"]

    resp = s.get(
        f"{prefix}/workflow/endpoint/", params={"workflow": workflow_id}, timeout=30
    )
    resp.raise_for_status()
    eps = resp.json()
    eps = eps if isinstance(eps, list) else eps.get("results", [])
    patched = 0
    for endpoint in eps:
        if endpoint.get("workflow") == workflow_id:
            resp = _patch(
                s,
                f"{prefix}/workflow/endpoint/{endpoint['id']}/",
                json={"connection_type": "API"},
            )
            assert resp.status_code == 200, f"patch endpoint: {resp.text}"
            patched += 1
    # A workflow always has exactly SOURCE+DESTINATION; a half-configured one
    # here would only surface downstream in an execute test.
    assert patched == 2, f"expected SOURCE+DESTINATION, patched {patched}: {eps}"

    resp = _post(
        s,
        f"{prefix}/tool_instance/",
        json={"workflow_id": workflow_id, "tool_id": prompt_registry_id},
    )
    assert resp.status_code == 201, f"attach tool: {resp.text}"

    return ProvisionedWorkflow(
        session=s,
        base=base,
        prefix=prefix,
        org_id=org_id,
        workflow_id=workflow_id,
        tool_id=tool_id,
        prompt_registry_id=prompt_registry_id,
        profile_id=profile_id,
        prompt_id=prompt_id,
        prompt_key=prompt_key,
    )