Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

29.0 KB
Raw Blame
"""Internal API Views for File Execution
Handles file execution related endpoints for internal services.
"""

import logging

from django.db import transaction
from rest_framework import status, viewsets
from rest_framework.decorators import action
from rest_framework.exceptions import APIException, NotFound
from rest_framework.response import Response
from rest_framework.views import APIView
from utils.organization_utils import filter_queryset_by_organization

from workflow_manager.endpoint_v2.dto import FileHash
from workflow_manager.file_execution.models import WorkflowFileExecution

# Import serializers from workflow_manager internal API
from workflow_manager.internal_serializers import (
    FileExecutionStatusUpdateSerializer,
    WorkflowFileExecutionSerializer,
)

logger = logging.getLogger(__name__)


class FileExecutionInternalViewSet(viewsets.ModelViewSet):
    """Internal API ViewSet for File Execution operations."""

    serializer_class = WorkflowFileExecutionSerializer
    lookup_field = "id"
    # OrganizationFilterBackend is off here. Unlike the other viewsets with
    # this pattern, get_object() below does not go through get_queryset() — it
    # builds its own queryset to skip the 3-table JOIN — so the two scope
    # independently, each via filter_queryset_by_organization, which fails
    # closed. X-Organization-ID is therefore required in practice on both
    # paths: a worker that omits it gets zero rows.
    skip_org_filter = True

    def get_object(self):
        """Retrieve a single file execution by PK, bypassing get_queryset().

        This avoids the expensive filtered queryset (3-table JOIN + query params)
        for single-object operations (retrieve, update, destroy).
        """
        pk = self.kwargs.get(self.lookup_field)
        queryset = WorkflowFileExecution.objects.filter(id=pk)
        queryset = filter_queryset_by_organization(
            queryset, self.request, "workflow_execution__workflow__organization"
        )
        try:
            obj = queryset.get()
        except WorkflowFileExecution.DoesNotExist as exc:
            raise NotFound(f"No file execution found with ID {pk}") from exc
        self.check_object_permissions(self.request, obj)
        return obj

    def get_queryset(self):
        """Get file executions filtered by organization context and query parameters.

        Only used for list() — single-object lookups go through get_object().
        """
        queryset = WorkflowFileExecution.objects.all()
        queryset = filter_queryset_by_organization(
            queryset, self.request, "workflow_execution__workflow__organization"
        )

        execution_id = self.request.query_params.get("execution_id")
        file_hash = self.request.query_params.get("file_hash")
        provider_file_uuid = self.request.query_params.get("provider_file_uuid")
        workflow_id = self.request.query_params.get("workflow_id")
        file_path = self.request.query_params.get("file_path")

        if execution_id:
            queryset = queryset.filter(workflow_execution_id=execution_id)

        if file_path:
            queryset = queryset.filter(file_path=file_path)

        # Use file_hash OR provider_file_uuid (not both) to match manager logic
        if file_hash:
            queryset = queryset.filter(file_hash=file_hash)
        elif provider_file_uuid:
            queryset = queryset.filter(provider_file_uuid=provider_file_uuid)

        if workflow_id:
            queryset = queryset.filter(workflow_execution__workflow_id=workflow_id)

        return queryset

    @action(detail=True, methods=["post"])
    def status(self, request, id=None):
        """Update file execution status."""
        try:
            file_execution = self.get_object()
            serializer = FileExecutionStatusUpdateSerializer(data=request.data)

            if serializer.is_valid():
                validated_data = serializer.validated_data

                # Update file execution using the model's update_status method
                file_execution.update_status(
                    status=validated_data["status"],
                    execution_error=validated_data.get("error_message"),
                    execution_time=validated_data.get("execution_time"),
                )

                logger.info(
                    f"Updated file execution {id} status to {validated_data['status']}"
                )

                # Return consistent dataclass response
                from unstract.core.data_models import FileExecutionStatusUpdateRequest

                response_data = FileExecutionStatusUpdateRequest(
                    status=file_execution.status,
                    error_message=file_execution.execution_error,
                    result=getattr(file_execution, "result", None),
                )

                return Response(
                    {
                        "status": "updated",
                        "file_execution_id": str(file_execution.id),
                        "data": response_data.to_dict(),
                    },
                    status=status.HTTP_200_OK,
                )

            return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)

        except APIException:
            raise
        except Exception as e:
            logger.error(f"Failed to update file execution status {id}: {str(e)}")
            return Response(
                {"error": "Failed to update file execution status", "detail": str(e)},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )

    def create(self, request, *args, **kwargs):
        """Create or get existing workflow file execution using existing manager method."""
        try:
            from workflow_manager.workflow_v2.models.execution import WorkflowExecution

            data = request.data
            execution_id = data.get("execution_id")
            file_hash_data = data.get("file_hash", {})
            workflow_id = data.get("workflow_id")

            if not execution_id:
                return Response(
                    {"error": "execution_id is required"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            # Get workflow execution with organization filtering
            try:
                workflow_execution = WorkflowExecution.objects.get(id=execution_id)
                # Verify organization access
                filter_queryset_by_organization(
                    WorkflowExecution.objects.filter(id=execution_id),
                    request,
                    "workflow__organization",
                ).get()
            except WorkflowExecution.DoesNotExist:
                return Response(
                    {"error": "WorkflowExecution not found or access denied"},
                    status=status.HTTP_404_NOT_FOUND,
                )

            # Convert request data to FileHash object that the manager expects
            file_hash = FileHash(
                file_path=file_hash_data.get("file_path", ""),
                file_name=file_hash_data.get("file_name", ""),
                source_connection_type=file_hash_data.get("source_connection_type", ""),
                file_hash=file_hash_data.get("file_hash"),
                file_size=file_hash_data.get("file_size"),
                provider_file_uuid=file_hash_data.get("provider_file_uuid"),
                mime_type=file_hash_data.get("mime_type"),
                fs_metadata=file_hash_data.get("fs_metadata"),
                file_destination=file_hash_data.get("file_destination"),
                is_executed=file_hash_data.get("is_executed", False),
                file_number=file_hash_data.get("file_number"),
            )

            # Determine if this is an API request (affects file_path handling in manager)
            is_api = file_hash_data.get("source_connection_type", "") == "API"

            # Use existing manager method - this handles get_or_create logic properly
            file_execution = WorkflowFileExecution.objects.get_or_create_file_execution(
                workflow_execution=workflow_execution, file_hash=file_hash, is_api=is_api
            )

            # Return single object (not list!) using serializer
            serializer = self.get_serializer(file_execution)
            response_data = serializer.data

            # ROOT CAUSE FIX: Ensure file_path is always present in API response
            # The backend model sets file_path to None for API files, but workers require it
            if not response_data.get("file_path") and file_hash.file_path:
                logger.info(
                    f"Backend stored null file_path for API file, including original: {file_hash.file_path}"
                )
                response_data["file_path"] = file_hash.file_path

            logger.info(
                f"Retrieved/created file execution {file_execution.id} for workflow {workflow_id}"
            )
            logger.debug(f"Response data: {response_data}")

            # Determine status code based on whether it was created or retrieved
            # Note: We can't easily tell if it was created or retrieved from the manager,
            # but 201 is fine for both cases in this context
            return Response(response_data, status=status.HTTP_201_CREATED)

        except Exception as e:
            logger.error(f"Failed to get/create file execution: {str(e)}")
            return Response(
                {"error": "Failed to get/create file execution", "detail": str(e)},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )

    @action(detail=True, methods=["patch"])
    def update_hash(self, request, id=None):
        """Update file execution with computed file hash."""
        try:
            file_execution = self.get_object()

            # Extract update data
            file_hash = request.data.get("file_hash")
            fs_metadata = request.data.get("fs_metadata")
            mime_type = request.data.get("mime_type")

            if not file_hash and not fs_metadata and not mime_type:
                return Response(
                    {"error": "file_hash, fs_metadata, or mime_type is required"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            # Use the model's update method for efficient field-specific updates
            file_execution.update(
                file_hash=file_hash, fs_metadata=fs_metadata, mime_type=mime_type
            )

            logger.info(
                f"Updated file execution {id} with file_hash: {file_hash[:16] if file_hash else 'none'}..."
            )

            # Return updated record
            serializer = self.get_serializer(file_execution)
            return Response(
                {
                    "status": "updated",
                    "file_execution_id": str(file_execution.id),
                    "data": serializer.data,
                },
                status=status.HTTP_200_OK,
            )

        except APIException:
            raise
        except Exception as e:
            logger.error(f"Failed to update file execution hash {id}: {str(e)}")
            return Response(
                {"error": "Failed to update file execution hash", "detail": str(e)},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )


class FileExecutionBatchCreateAPIView(APIView):
    """Internal API endpoint for creating multiple file executions in a single batch."""

    def post(self, request):
        """Create multiple file executions in a single batch request."""
        try:
            file_executions = request.data.get("file_executions", [])

            if not file_executions:
                return Response(
                    {"error": "file_executions list is required"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            successful_creations = []
            failed_creations = []

            with transaction.atomic():
                for file_execution_data in file_executions:
                    try:
                        from workflow_manager.workflow_v2.models.execution import (
                            WorkflowExecution,
                        )

                        execution_id = file_execution_data.get("execution_id")
                        file_hash_data = file_execution_data.get("file_hash", {})

                        if not execution_id:
                            failed_creations.append(
                                {
                                    "file_name": file_hash_data.get(
                                        "file_name", "unknown"
                                    ),
                                    "error": "execution_id is required",
                                }
                            )
                            continue

                        # Get workflow execution with organization filtering
                        try:
                            workflow_execution = WorkflowExecution.objects.get(
                                id=execution_id
                            )
                            # Verify organization access
                            filter_queryset_by_organization(
                                WorkflowExecution.objects.filter(id=execution_id),
                                request,
                                "workflow__organization",
                            ).get()
                        except WorkflowExecution.DoesNotExist:
                            failed_creations.append(
                                {
                                    "file_name": file_hash_data.get(
                                        "file_name", "unknown"
                                    ),
                                    "error": "WorkflowExecution not found or access denied",
                                }
                            )
                            continue

                        # Convert request data to FileHash object
                        file_hash = FileHash(
                            file_path=file_hash_data.get("file_path", ""),
                            file_name=file_hash_data.get("file_name", ""),
                            source_connection_type=file_hash_data.get(
                                "source_connection_type", ""
                            ),
                            file_hash=file_hash_data.get("file_hash"),
                            file_size=file_hash_data.get("file_size"),
                            provider_file_uuid=file_hash_data.get("provider_file_uuid"),
                            mime_type=file_hash_data.get("mime_type"),
                            fs_metadata=file_hash_data.get("fs_metadata"),
                            file_destination=file_hash_data.get("file_destination"),
                            is_executed=file_hash_data.get("is_executed", False),
                            file_number=file_hash_data.get("file_number"),
                        )

                        # Determine if this is an API request
                        is_api = file_hash_data.get("source_connection_type", "") == "API"

                        # Use existing manager method
                        file_execution = (
                            WorkflowFileExecution.objects.get_or_create_file_execution(
                                workflow_execution=workflow_execution,
                                file_hash=file_hash,
                                is_api=is_api,
                            )
                        )

                        # ROOT CAUSE FIX: Ensure file_path is always present in batch response
                        # The backend model sets file_path to None for API files, but workers require it
                        response_file_path = file_execution.file_path
                        if not response_file_path and file_hash.file_path:
                            response_file_path = file_hash.file_path

                        successful_creations.append(
                            {
                                "id": str(file_execution.id),
                                "file_name": file_execution.file_name,
                                "file_path": response_file_path,
                                "status": file_execution.status,
                            }
                        )

                    except Exception as e:
                        failed_creations.append(
                            {
                                "file_name": file_execution_data.get("file_hash", {}).get(
                                    "file_name", "unknown"
                                ),
                                "error": str(e),
                            }
                        )

            logger.info(
                f"Batch file execution creation: {len(successful_creations)} successful, {len(failed_creations)} failed"
            )

            return Response(
                {
                    "successful_creations": successful_creations,
                    "failed_creations": failed_creations,
                    "total_processed": len(file_executions),
                },
                status=status.HTTP_201_CREATED,
            )

        except Exception as e:
            logger.error(f"Failed to process batch file execution creation: {str(e)}")
            return Response(
                {
                    "error": "Failed to process batch file execution creation",
                    "detail": str(e),
                },
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )


class FileExecutionBatchStatusUpdateAPIView(APIView):
    """Internal API endpoint for updating multiple file execution statuses in a single batch."""

    def post(self, request):
        """Update multiple file execution statuses in a single batch request."""
        try:
            status_updates = request.data.get("status_updates", [])

            if not status_updates:
                return Response(
                    {"error": "status_updates list is required"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            successful_updates = []
            failed_updates = []

            with transaction.atomic():
                for update_data in status_updates:
                    try:
                        file_execution_id = update_data.get("file_execution_id")
                        status_value = update_data.get("status")

                        if not file_execution_id or not status_value:
                            failed_updates.append(
                                {
                                    "file_execution_id": file_execution_id,
                                    "error": "file_execution_id and status are required",
                                }
                            )
                            continue

                        # Get file execution with organization filtering
                        base_queryset = WorkflowFileExecution.objects.all()
                        base_queryset = filter_queryset_by_organization(
                            base_queryset,
                            request,
                            "workflow_execution__workflow__organization",
                        )

                        try:
                            file_execution = base_queryset.get(id=file_execution_id)
                        except WorkflowFileExecution.DoesNotExist:
                            failed_updates.append(
                                {
                                    "file_execution_id": file_execution_id,
                                    "error": "WorkflowFileExecution not found",
                                }
                            )
                            continue

                        # Update file execution using the model's update_status method
                        file_execution.update_status(
                            status=status_value,
                            execution_error=update_data.get("error_message"),
                            execution_time=update_data.get("execution_time"),
                        )

                        successful_updates.append(
                            {
                                "file_execution_id": str(file_execution.id),
                                "status": file_execution.status,
                                "file_name": file_execution.file_name,
                            }
                        )

                    except Exception as e:
                        failed_updates.append(
                            {"file_execution_id": file_execution_id, "error": str(e)}
                        )

            logger.info(
                f"Batch file execution status update: {len(successful_updates)} successful, {len(failed_updates)} failed"
            )

            return Response(
                {
                    "successful_updates": successful_updates,
                    "failed_updates": failed_updates,
                    "total_processed": len(status_updates),
                }
            )

        except Exception as e:
            logger.error(
                f"Failed to process batch file execution status update: {str(e)}"
            )
            return Response(
                {
                    "error": "Failed to process batch file execution status update",
                    "detail": str(e),
                },
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )


class FileExecutionBatchHashUpdateAPIView(APIView):
    """Internal API endpoint for updating multiple file execution hashes in a single batch."""

    def post(self, request):
        """Update multiple file execution hashes in a single batch request."""
        try:
            hash_updates = request.data.get("hash_updates", [])

            if not hash_updates:
                return Response(
                    {"error": "hash_updates list is required"},
                    status=status.HTTP_400_BAD_REQUEST,
                )

            successful_updates = []
            failed_updates = []

            with transaction.atomic():
                for update_data in hash_updates:
                    try:
                        file_execution_id = update_data.get("file_execution_id")
                        file_hash = update_data.get("file_hash")

                        if not file_execution_id or not file_hash:
                            failed_updates.append(
                                {
                                    "file_execution_id": file_execution_id,
                                    "error": "file_execution_id and file_hash are required",
                                }
                            )
                            continue

                        # Get file execution with organization filtering
                        base_queryset = WorkflowFileExecution.objects.all()
                        base_queryset = filter_queryset_by_organization(
                            base_queryset,
                            request,
                            "workflow_execution__workflow__organization",
                        )

                        try:
                            file_execution = base_queryset.get(id=file_execution_id)
                        except WorkflowFileExecution.DoesNotExist:
                            failed_updates.append(
                                {
                                    "file_execution_id": file_execution_id,
                                    "error": "WorkflowFileExecution not found",
                                }
                            )
                            continue

                        # Update file execution hash using the model's update method
                        file_execution.update(
                            file_hash=file_hash,
                            fs_metadata=update_data.get("fs_metadata"),
                        )

                        successful_updates.append(
                            {
                                "file_execution_id": str(file_execution.id),
                                "file_hash": file_hash[:16] + "..."
                                if file_hash
                                else None,
                                "file_name": file_execution.file_name,
                            }
                        )

                    except Exception as e:
                        failed_updates.append(
                            {"file_execution_id": file_execution_id, "error": str(e)}
                        )

            logger.info(
                f"Batch file execution hash update: {len(successful_updates)} successful, {len(failed_updates)} failed"
            )

            return Response(
                {
                    "successful_updates": successful_updates,
                    "failed_updates": failed_updates,
                    "total_processed": len(hash_updates),
                }
            )

        except Exception as e:
            logger.error(f"Failed to process batch file execution hash update: {str(e)}")
            return Response(
                {
                    "error": "Failed to process batch file execution hash update",
                    "detail": str(e),
                },
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )


class FileExecutionMetricsAPIView(APIView):
    """Internal API endpoint for getting file execution metrics."""

    def get(self, request):
        """Get file execution metrics with optional filtering."""
        try:
            # Get query parameters
            start_date = request.query_params.get("start_date")
            end_date = request.query_params.get("end_date")
            workflow_id = request.query_params.get("workflow_id")
            execution_id = request.query_params.get("execution_id")
            status = request.query_params.get("status")

            # Build base queryset with organization filtering
            file_executions = WorkflowFileExecution.objects.all()
            file_executions = filter_queryset_by_organization(
                file_executions, request, "workflow_execution__workflow__organization"
            )

            # Apply filters
            if start_date:
                from datetime import datetime

                file_executions = file_executions.filter(
                    created_at__gte=datetime.fromisoformat(start_date)
                )
            if end_date:
                from datetime import datetime

                file_executions = file_executions.filter(
                    created_at__lte=datetime.fromisoformat(end_date)
                )
            if workflow_id:
                file_executions = file_executions.filter(
                    workflow_execution__workflow_id=workflow_id
                )
            if execution_id:
                file_executions = file_executions.filter(
                    workflow_execution_id=execution_id
                )
            if status:
                file_executions = file_executions.filter(status=status)

            # Calculate metrics
            from django.db.models import Avg, Count, Sum

            total_file_executions = file_executions.count()

            # Status breakdown
            status_counts = file_executions.values("status").annotate(count=Count("id"))
            status_breakdown = {item["status"]: item["count"] for item in status_counts}

            # Success rate
            completed_count = status_breakdown.get("COMPLETED", 0)
            success_rate = (
                (completed_count / total_file_executions)
                if total_file_executions > 0
                else 0
            )

            # Average execution time
            avg_execution_time = (
                file_executions.aggregate(avg_time=Avg("execution_time"))["avg_time"] or 0
            )

            # File size statistics
            total_file_size = (
                file_executions.aggregate(total_size=Sum("file_size"))["total_size"] or 0
            )

            avg_file_size = (
                file_executions.aggregate(avg_size=Avg("file_size"))["avg_size"] or 0
            )

            metrics = {
                "total_file_executions": total_file_executions,
                "status_breakdown": status_breakdown,
                "success_rate": success_rate,
                "average_execution_time": avg_execution_time,
                "total_file_size": total_file_size,
                "average_file_size": avg_file_size,
                "filters_applied": {
                    "start_date": start_date,
                    "end_date": end_date,
                    "workflow_id": workflow_id,
                    "execution_id": execution_id,
                    "status": status,
                },
            }

            logger.info(
                f"Generated file execution metrics: {total_file_executions} executions, {success_rate:.2%} success rate"
            )

            return Response(metrics)

        except Exception as e:
            logger.error(f"Failed to get file execution metrics: {str(e)}")
            return Response(
                {"error": "Failed to get file execution metrics", "detail": str(e)},
                status=status.HTTP_500_INTERNAL_SERVER_ERROR,
            )