Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

10.1 KB
Raw Blame
from abc import ABC, abstractmethod
from dataclasses import asdict, dataclass, field
from typing import Any, Literal
from urllib.parse import urlencode, urljoin

from django.conf import settings
from pipeline_v2.models import Pipeline
from utils.request import HTTPMethod

from api_v2.constants import ApiExecution
from api_v2.models import APIDeployment
from api_v2.postman_collection.constants import CollectionKey


@dataclass
class HeaderItem:
    key: str
    value: str


@dataclass
class ScriptItem:
    exec: list[str]
    type: Literal["text/javascript"] = "text/javascript"


@dataclass
class EventItem:
    listen: Literal["prerequest", "test"]
    script: ScriptItem


@dataclass
class VariableItem:
    key: str
    value: str


@dataclass
class FormDataItem:
    key: str
    type: str
    src: str | None = None
    value: str | None = None
    description: str | None = None

    def __post_init__(self) -> None:
        if self.type == "file":
            if self.src is None:
                raise ValueError("src must be provided for type 'file'")
        elif self.type == "text":
            if self.value is None:
                raise ValueError("value must be provided for type 'text'")
        else:
            raise ValueError(f"Unsupported type for form data: {self.type}")


@dataclass
class BodyItem:
    formdata: list[FormDataItem]
    mode: str = "formdata"


@dataclass
class RequestItem:
    method: HTTPMethod
    url: str
    header: list[HeaderItem]
    body: BodyItem | None = None


@dataclass
class PostmanItem:
    name: str
    request: RequestItem
    event: list[EventItem] = field(default_factory=list)


@dataclass
class PostmanInfo:
    name: str = "Unstract's API deployment"
    schema: str = CollectionKey.POSTMAN_COLLECTION_V210
    description: str = "Contains APIs meant for using the deployed Unstract API"


class APIBase(ABC):
    @abstractmethod
    def get_form_data_items(self) -> list[FormDataItem]:
        pass

    def get_collection_variables(self) -> list[VariableItem]:
        """Collection-level variables; only needed when a request
        references them.
        """
        return []

    @abstractmethod
    def get_api_endpoint(self) -> str:
        pass

    @abstractmethod
    def get_postman_items(self) -> list[PostmanItem]:
        pass

    @abstractmethod
    def get_api_key(self) -> str:
        pass

    def get_execute_body(self) -> BodyItem:
        form_data_items = self.get_form_data_items()
        return BodyItem(formdata=form_data_items)

    def get_create_api_request(self) -> RequestItem:
        header_list = [
            HeaderItem(key="Authorization", value=f"Bearer {self.get_api_key()}")
        ]
        abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, self.get_api_endpoint())
        return RequestItem(
            method=HTTPMethod.POST,
            header=header_list,
            body=self.get_execute_body(),
            url=abs_api_endpoint,
        )


@dataclass
class APIDeploymentDto(APIBase):
    display_name: str
    description: str
    api_endpoint: str
    api_key: str

    def get_postman_info(self) -> PostmanInfo:
        return PostmanInfo(name=self.display_name, description=self.description)

    def get_form_data_items(self) -> list[FormDataItem]:
        return [
            FormDataItem(
                key=ApiExecution.FILES_FORM_DATA, type="file", src="/path_to_file"
            ),
            FormDataItem(
                key=ApiExecution.TIMEOUT_FORM_DATA,
                type="text",
                value=ApiExecution.MAXIMUM_TIMEOUT_IN_SEC,
            ),
            FormDataItem(key=ApiExecution.INCLUDE_METADATA, type="text", value="False"),
            FormDataItem(key=ApiExecution.INCLUDE_METRICS, type="text", value="False"),
            FormDataItem(
                key=ApiExecution.LLM_PROFILE_ID,
                type="text",
                value="",
                description="Optional: UUID of the LLM profile to override default settings",
            ),
        ]

    def get_api_key(self) -> str:
        return self.api_key

    def get_api_endpoint(self) -> str:
        return self.api_endpoint

    def _get_status_api_request(self) -> RequestItem:
        header_list = [HeaderItem(key="Authorization", value=f"Bearer {self.api_key}")]
        status_query_param = {
            "execution_id": CollectionKey.STATUS_EXEC_ID_VARIABLE,
            ApiExecution.INCLUDE_METADATA: "False",
            ApiExecution.INCLUDE_METRICS: "False",
        }
        # Keep {{...}} unescaped so Postman resolves the collection variable
        status_query_str = urlencode(status_query_param, safe="{}")
        abs_api_endpoint = urljoin(settings.WEB_APP_ORIGIN_URL, self.api_endpoint)
        status_url = urljoin(abs_api_endpoint, "?" + status_query_str)
        return RequestItem(method=HTTPMethod.GET, header=header_list, url=status_url)

    def get_collection_variables(self) -> list[VariableItem]:
        return [
            VariableItem(
                key=CollectionKey.EXEC_ID_VARIABLE_NAME,
                value=CollectionKey.STATUS_EXEC_ID_DEFAULT,
            )
        ]

    def _get_execute_capture_event(self) -> EventItem:
        """Post-response script that stores the execution_id from the execute
        response into a collection variable, so the status request can use it
        without manual copy-pasting.

        This is Postman's ``test`` hook (the emitted event uses
        ``listen="test"``). It assumes the execute response shape
        ``{"message": {"execution_id": ...}}``; if the body is non-JSON or
        lacks that field it captures nothing and resets the variable to the
        default sentinel so a stale id from a previous run is never reused.
        """
        return EventItem(
            listen="test",
            script=ScriptItem(
                exec=[
                    "let response = null;",
                    "try {",
                    "    response = pm.response.json();",
                    "} catch (error) {",
                    "    // Non-JSON response (e.g. gateway error); nothing to capture",
                    "}",
                    "if (response && response.message && response.message.execution_id) {",  # noqa: E501
                    f'    pm.collectionVariables.set("{CollectionKey.EXEC_ID_VARIABLE_NAME}", response.message.execution_id);',  # noqa: E501
                    "} else {",
                    f'    pm.collectionVariables.set("{CollectionKey.EXEC_ID_VARIABLE_NAME}", "{CollectionKey.STATUS_EXEC_ID_DEFAULT}");',  # noqa: E501
                    '    console.warn("No execution_id in execute response; not reusing a stale value. Status:", pm.response.code);',  # noqa: E501
                    "}",
                ]
            ),
        )

    def get_postman_items(self) -> list[PostmanItem]:
        postman_item_list = [
            PostmanItem(
                name=CollectionKey.EXECUTE_API_KEY,
                request=self.get_create_api_request(),
                event=[self._get_execute_capture_event()],
            ),
            PostmanItem(
                name=CollectionKey.STATUS_API_KEY,
                request=self._get_status_api_request(),
            ),
        ]
        return postman_item_list


@dataclass
class PipelineDto(APIBase):
    pipeline_name: str
    api_endpoint: str
    api_key: str

    def get_postman_info(self) -> PostmanInfo:
        return PostmanInfo(name=self.pipeline_name, description="")

    def get_form_data_items(self) -> list[FormDataItem]:
        return []

    def get_api_endpoint(self) -> str:
        return self.api_endpoint

    def get_api_key(self) -> str:
        return self.api_key

    def get_postman_items(self) -> list[PostmanItem]:
        postman_item_list = [
            PostmanItem(
                name=CollectionKey.EXECUTE_PIPELINE_API_KEY,
                request=self.get_create_api_request(),
            )
        ]
        return postman_item_list


@dataclass
class PostmanCollection:
    info: PostmanInfo
    item: list[PostmanItem] = field(default_factory=list)
    variable: list[VariableItem] = field(default_factory=list)

    @classmethod
    def create(
        cls,
        instance: APIDeployment | Pipeline,
        api_key: str = CollectionKey.AUTH_QUERY_PARAM_DEFAULT,
    ) -> "PostmanCollection":
        """Creates a PostmanCollection instance.

        This instance can help represent Postman collections (v2 format) that
        can be used to easily invoke workflows deployed as APIs

        Args:
            instance (APIDeployment): API deployment to generate collection for
            api_key (str, optional): Active API key used to authenticate requests for
                deployed APIs. Defaults to CollectionKey.AUTH_QUERY_PARAM_DEFAULT.

        Returns:
            PostmanCollection: Instance representing PostmanCollection
        """
        data_object: APIBase
        if isinstance(instance, APIDeployment):
            data_object = APIDeploymentDto(
                display_name=instance.display_name,
                description=instance.description,
                api_endpoint=instance.api_endpoint,
                api_key=api_key,
            )
        elif isinstance(instance, Pipeline):
            data_object = PipelineDto(
                pipeline_name=instance.pipeline_name,
                api_endpoint=instance.api_endpoint,
                api_key=api_key,
            )
        postman_info: PostmanInfo = data_object.get_postman_info()
        postman_item_list = data_object.get_postman_items()
        return cls(
            info=postman_info,
            item=postman_item_list,
            variable=data_object.get_collection_variables(),
        )

    def to_dict(self) -> dict[str, Any]:
        """Convert PostmanCollection instance to a dict.

        Returns:
            dict[str, Any]: PostmanCollection as a dict
        """
        collection_dict = asdict(self)
        # Omit empty event blocks; items without scripts (e.g. pipelines)
        # should not carry an "event" key at all.
        for item in collection_dict.get("item", []):
            if not item.get("event"):
                item.pop("event", None)
        return collection_dict