Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

16.4 KB
Raw Blame
"""Shared Data Models for Worker Services

This module contains dataclass definitions used across worker services
to ensure type safety and prevent dictionary usage for structured data.
"""

from dataclasses import dataclass
from datetime import datetime
from typing import Any
from uuid import UUID

from ..enums import (
    BatchOperationType,
    CircuitBreakerState,
    ConnectionType,
    EndpointType,
    FileOperationType,
    HTTPMethod,
    LogLevel,
    NotificationPlatform,
    TaskStatus,
    ToolOutputType,
)


@dataclass
class WorkflowExecutionData:
    """Workflow execution data structure."""

    id: str | UUID
    workflow_id: str | UUID
    status: TaskStatus
    created_at: datetime
    updated_at: datetime
    execution_method: str | None = None
    total_files: int | None = None
    completed_files: int | None = None
    failed_files: int | None = None
    error_message: str | None = None
    execution_time: float | None = None
    attempts: int | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "workflow_id": str(self.workflow_id),
            "status": self.status.value,
            "created_at": self.created_at.isoformat(),
            "updated_at": self.updated_at.isoformat(),
            "execution_method": self.execution_method,
            "total_files": self.total_files,
            "completed_files": self.completed_files,
            "failed_files": self.failed_files,
            "error_message": self.error_message,
            "execution_time": self.execution_time,
            "attempts": self.attempts,
        }


@dataclass
class WorkflowDefinition:
    """Workflow definition data structure."""

    id: str | UUID
    workflow_name: str
    workflow_type: str
    description: str | None = None
    created_at: datetime | None = None
    updated_at: datetime | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "workflow_name": self.workflow_name,
            "workflow_type": self.workflow_type,
            "description": self.description,
            "created_at": self.created_at.isoformat() if self.created_at else None,
            "updated_at": self.updated_at.isoformat() if self.updated_at else None,
        }


@dataclass
class FileExecutionData:
    """File execution data structure."""

    id: str | UUID
    workflow_execution_id: str | UUID
    file_path: str
    file_name: str
    status: TaskStatus
    created_at: datetime
    updated_at: datetime
    file_size: int | None = None
    mime_type: str | None = None
    file_hash: str | None = None
    processing_time: float | None = None
    error_message: str | None = None
    result_data: dict[str, Any] | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "workflow_execution_id": str(self.workflow_execution_id),
            "file_path": self.file_path,
            "file_name": self.file_name,
            "status": self.status.value,
            "created_at": self.created_at.isoformat(),
            "updated_at": self.updated_at.isoformat(),
            "file_size": self.file_size,
            "mime_type": self.mime_type,
            "file_hash": self.file_hash,
            "processing_time": self.processing_time,
            "error_message": self.error_message,
            "result_data": self.result_data,
        }


@dataclass
class ToolExecutionData:
    """Tool execution data structure."""

    id: str | UUID
    tool_name: str
    tool_version: str
    status: TaskStatus
    output_type: ToolOutputType
    created_at: datetime
    updated_at: datetime
    input_data: dict[str, Any] | None = None
    output_data: dict[str, Any] | None = None
    execution_time: float | None = None
    error_message: str | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "tool_name": self.tool_name,
            "tool_version": self.tool_version,
            "status": self.status.value,
            "output_type": self.output_type.value,
            "created_at": self.created_at.isoformat(),
            "updated_at": self.updated_at.isoformat(),
            "input_data": self.input_data,
            "output_data": self.output_data,
            "execution_time": self.execution_time,
            "error_message": self.error_message,
        }


@dataclass
class WebhookDeliveryData:
    """Webhook delivery data structure."""

    id: str | UUID
    url: str
    method: HTTPMethod
    platform: NotificationPlatform
    status: TaskStatus
    created_at: datetime
    updated_at: datetime
    payload: dict[str, Any] | None = None
    headers: dict[str, str] | None = None
    response_code: int | None = None
    response_data: dict[str, Any] | None = None
    attempts: int | None = None
    next_retry_at: datetime | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "url": self.url,
            "method": self.method.value,
            "platform": self.platform.value,
            "status": self.status.value,
            "created_at": self.created_at.isoformat(),
            "updated_at": self.updated_at.isoformat(),
            "payload": self.payload,
            "headers": self.headers,
            "response_code": self.response_code,
            "response_data": self.response_data,
            "attempts": self.attempts,
            "next_retry_at": self.next_retry_at.isoformat()
            if self.next_retry_at
            else None,
        }


@dataclass
class EndpointConfiguration:
    """Endpoint configuration data structure."""

    id: str | UUID
    endpoint_type: EndpointType
    connection_type: ConnectionType
    configuration: dict[str, Any]
    name: str | None = None
    description: str | None = None
    is_active: bool = True

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "endpoint_type": self.endpoint_type.value,
            "connection_type": self.connection_type.value,
            "configuration": self.configuration,
            "name": self.name,
            "description": self.description,
            "is_active": self.is_active,
        }


@dataclass
class BatchOperationRequest:
    """Batch operation request data structure."""

    operation_type: BatchOperationType
    items: list[dict[str, Any]]
    organization_id: str | None = None
    batch_size: int = 100
    parallel_processing: bool = True

    def to_dict(self) -> dict[str, Any]:
        # Use 'updates' field name for status update operations to match backend API
        items_field_name = (
            "updates"
            if self.operation_type == BatchOperationType.STATUS_UPDATE
            else "items"
        )

        return {
            "operation_type": self.operation_type.value,
            items_field_name: self.items,
            "organization_id": self.organization_id,
            "batch_size": self.batch_size,
            "parallel_processing": self.parallel_processing,
        }


@dataclass
class BatchOperationResponse:
    """Batch operation response data structure."""

    operation_id: str
    total_items: int
    successful_items: int
    failed_items: int
    status: TaskStatus
    results: list[dict[str, Any]]
    errors: list[dict[str, Any]]
    execution_time: float | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "operation_id": self.operation_id,
            "total_items": self.total_items,
            "successful_items": self.successful_items,
            "failed_items": self.failed_items,
            "status": self.status.value,
            "results": self.results,
            "errors": self.errors,
            "execution_time": self.execution_time,
        }


@dataclass
class FileHistoryData:
    """File history data structure."""

    id: str | UUID
    file_execution_id: str | UUID
    operation_type: FileOperationType
    timestamp: datetime
    file_path: str
    file_name: str
    file_size: int | None = None
    status: TaskStatus | None = None
    metadata: dict[str, Any] | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "file_execution_id": str(self.file_execution_id),
            "operation_type": self.operation_type.value,
            "timestamp": self.timestamp.isoformat(),
            "file_path": self.file_path,
            "file_name": self.file_name,
            "file_size": self.file_size,
            "status": self.status.value if self.status else None,
            "metadata": self.metadata,
        }


@dataclass
class CircuitBreakerStatus:
    """Circuit breaker status data structure."""

    state: CircuitBreakerState
    failure_count: int
    last_failure_time: datetime | None = None
    next_attempt_time: datetime | None = None
    success_count: int = 0
    total_requests: int = 0

    def to_dict(self) -> dict[str, Any]:
        return {
            "state": self.state.value,
            "failure_count": self.failure_count,
            "last_failure_time": self.last_failure_time.isoformat()
            if self.last_failure_time
            else None,
            "next_attempt_time": self.next_attempt_time.isoformat()
            if self.next_attempt_time
            else None,
            "success_count": self.success_count,
            "total_requests": self.total_requests,
        }


@dataclass
class LogEntry:
    """Log entry data structure."""

    timestamp: datetime
    level: LogLevel
    message: str
    logger_name: str
    module: str | None = None
    function: str | None = None
    line_number: int | None = None
    context: dict[str, Any] | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "timestamp": self.timestamp.isoformat(),
            "level": self.level.value,
            "message": self.message,
            "logger_name": self.logger_name,
            "module": self.module,
            "function": self.function,
            "line_number": self.line_number,
            "context": self.context,
        }


@dataclass
class APIResponse:
    """Generic API response data structure."""

    success: bool
    data: dict[str, Any] | None = None
    error: str | None = None
    status_code: int | None = None
    timestamp: datetime | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "success": self.success,
            "data": self.data,
            "error": self.error,
            "status_code": self.status_code,
            "timestamp": self.timestamp.isoformat() if self.timestamp else None,
        }


@dataclass
class StatusUpdateRequest:
    """Status update request data structure."""

    id: str | UUID
    status: TaskStatus
    error_message: str | None = None
    metadata: dict[str, Any] | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "id": str(self.id),
            "status": self.status.value,
            "error_message": self.error_message,
            "metadata": self.metadata,
        }


@dataclass
class MetricsData:
    """Metrics data structure."""

    timestamp: datetime
    metric_name: str
    value: int | float
    unit: str
    tags: dict[str, str] | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "timestamp": self.timestamp.isoformat(),
            "metric_name": self.metric_name,
            "value": self.value,
            "unit": self.unit,
            "tags": self.tags,
        }


@dataclass
class WorkerTaskResponse:
    """Generic worker task response structure."""

    status: str
    execution_id: str | UUID
    workflow_id: str | UUID
    task_id: str | None = None
    execution_time: float | None = None
    success: bool = True
    error: str | None = None
    is_general_workflow: bool | None = None
    pipeline_id: str | UUID | None = None

    def to_dict(self) -> dict[str, Any]:
        return {
            "status": self.status,
            "execution_id": str(self.execution_id),
            "workflow_id": str(self.workflow_id),
            "task_id": self.task_id,
            "execution_time": self.execution_time,
            "success": self.success,
            "error": self.error,
            "is_general_workflow": self.is_general_workflow,
            "pipeline_id": str(self.pipeline_id) if self.pipeline_id else None,
        }

    @classmethod
    def success_response(
        cls,
        execution_id: str | UUID,
        workflow_id: str | UUID,
        task_id: str | None = None,
        execution_time: float | None = None,
        pipeline_id: str | UUID | None = None,
    ) -> "WorkerTaskResponse":
        """Create a success response."""
        return cls(
            status="success",
            execution_id=execution_id,
            workflow_id=workflow_id,
            task_id=task_id,
            execution_time=execution_time,
            success=True,
            pipeline_id=pipeline_id,
        )

    @classmethod
    def error_response(
        cls,
        execution_id: str | UUID,
        workflow_id: str | UUID,
        error: str,
        task_id: str | None = None,
        execution_time: float | None = None,
    ) -> "WorkerTaskResponse":
        """Create an error response."""
        return cls(
            status="error",
            execution_id=execution_id,
            workflow_id=workflow_id,
            task_id=task_id,
            execution_time=execution_time,
            success=False,
            error=error,
        )


def validate_uuid(value: str | UUID | None) -> UUID | None:
    if value is None:
        return None
    if isinstance(value, UUID):
        return value
    try:
        return UUID(str(value))
    except (ValueError, AttributeError, TypeError):
        return None


@dataclass
class CallbackTaskData:
    """Callback task data structure for worker callbacks."""

    execution_id: str | UUID
    organization_id: str
    pipeline_id: str | UUID | None = None

    def __post_init__(self):
        self.execution_id = validate_uuid(self.execution_id)
        # organization_id is a required string identifier (e.g. "org_xxx"), not a UUID
        if not self.organization_id:
            raise ValueError("organization_id is required for CallbackTaskData")
        self.organization_id = str(self.organization_id)
        self.pipeline_id = validate_uuid(self.pipeline_id)

    def to_dict(self) -> dict[str, Any]:
        data = {
            "execution_id": str(self.execution_id) if self.execution_id else None,
            "organization_id": self.organization_id,
        }
        if self.pipeline_id:
            data["pipeline_id"] = str(self.pipeline_id)
        return data


@dataclass
class WorkflowExecutionStatusUpdate:
    """Workflow execution status update data structure."""

    execution_id: str | UUID
    status: str
    execution_time: float | None = None
    total_files: int | None = None

    def to_dict(self) -> dict[str, Any]:
        data = {
            "execution_id": str(self.execution_id),
            "status": self.status,
        }
        if self.execution_time is not None:
            data["execution_time"] = self.execution_time
        if self.total_files is not None:
            data["total_files"] = self.total_files
        return data


# Utility functions for data model conversion
def dict_to_dataclass(data: dict[str, Any], dataclass_type):
    """Convert dictionary to dataclass instance."""
    if not isinstance(data, dict):
        return data

    # Get field names and types from dataclass
    from dataclasses import fields

    if not hasattr(dataclass_type, "__dataclass_fields__"):
        return data

    field_types = {field.name: field.type for field in fields(dataclass_type)}
    kwargs = {}

    for field_name, field_type in field_types.items():
        if field_name in data:
            value = data[field_name]

            # Handle datetime conversion
            if field_type == datetime and isinstance(value, str):
                kwargs[field_name] = datetime.fromisoformat(value.replace("Z", "+00:00"))
            # Handle UUID conversion
            elif field_type in [UUID, str | UUID] and isinstance(value, str):
                kwargs[field_name] = value
            # Handle enum conversion
            elif hasattr(field_type, "__bases__") and any(
                base.__name__ == "Enum" for base in field_type.__bases__
            ):
                kwargs[field_name] = field_type(value)
            else:
                kwargs[field_name] = value

    return dataclass_type(**kwargs)