Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

68.7 KB
Raw Blame
"""Backward Compatibility Facade for Internal API Client

This facade provides backward compatibility for existing code that imports from
the original api_client.py. It delegates calls to the specialized modular clients
while maintaining the same interface.

This maintains the original InternalAPIClient interface by composing all the
specialized clients (ExecutionAPIClient, FileAPIClient, etc.) into a single
unified interface.
"""

import logging
import threading
import time
import uuid
from typing import Any
from uuid import UUID

# Manual review functionality loaded via plugin registry
from client_plugin_registry import get_client_plugin
from shared.cache import CachedAPIClientMixin, with_cache
from shared.cache.cache_types import CacheType
from shared.clients import (
    BaseAPIClient,
    ExecutionAPIClient,
    FileAPIClient,
    OrganizationAPIClient,
    ToolAPIClient,
    UsageAPIClient,
    WebhookAPIClient,
    WorkflowAPIClient,
)

# Import exceptions from base client
# Re-export exceptions for backward compatibility
from shared.clients.base_client import (
    APIRequestError,
    AuthenticationError,
    InternalAPIClientError,
)
from shared.clients.manual_review_stub import ManualReviewNullClient
from shared.data.response_models import APIResponse
from shared.enums import HTTPMethod
from shared.infrastructure.config.worker_config import WorkerConfig

# Import new API response dataclasses for type safety
from shared.models.api_responses import (
    FileBatchResponse,
    FileHistoryResponse,
    ToolInstancesResponse,
    WorkflowDefinitionResponse,
    WorkflowExecutionResponse,
)

# Import execution models for type-safe execution contexts
# Import notification models for type-safe notifications
from shared.models.notification_models import (
    WebhookNotificationRequest,
)
from shared.models.scheduler_models import SchedulerExecutionResult

from unstract.core.data_models import (
    ExecutionStatus,
    FileExecutionCreateRequest,
    FileExecutionStatusUpdateRequest,
    FileHashData,
    UsageResponseData,
    WorkflowDefinitionResponseData,
    WorkflowEndpointConfigResponseData,
    WorkflowFileExecutionData,
)

logger = logging.getLogger(__name__)


class InternalAPIClient(CachedAPIClientMixin):
    """Backward compatibility facade for the original monolithic InternalAPIClient.

    This class provides the same interface as the original InternalAPIClient
    but delegates all operations to the specialized modular clients with caching support.
    This ensures existing code continues to work without modification while benefiting from
    both the improved modular architecture and caching for better performance.
    """

    # Class-level shared base client for singleton pattern
    _shared_base_client = None
    _shared_session = None
    _initialization_count = 0

    # Task counter for periodic singleton reset (FR-3.2)
    _task_counter: int = 0
    _task_counter_lock: threading.Lock = threading.Lock()
    _last_reset_time: float | None = None
    _cached_reset_threshold: int | None = None

    def __init__(self, config: WorkerConfig | None = None):
        """Initialize the facade with all specialized clients and caching.

        Args:
            config: Worker configuration. If None, uses default config.
        """
        self.config = config or WorkerConfig()

        # Initialize caching (parent class)
        super().__init__()

        # Initialize core clients
        self._initialize_core_clients()

        # Initialize plugin clients
        self._initialize_plugin_clients()

        # Setup direct access references
        self._setup_direct_access_references()

        logger.info(
            "Initialized InternalAPIClient facade with modular architecture and caching"
        )

    def _initialize_core_clients(self) -> None:
        """Initialize all core API clients with performance optimizations."""
        if self.config.enable_api_client_singleton:
            self._initialize_core_clients_optimized()
        else:
            self._initialize_core_clients_traditional()

    def _initialize_core_clients_optimized(self) -> None:
        """Initialize clients using GIL-safe singleton pattern for better performance."""
        InternalAPIClient._initialization_count += 1

        # Create or reuse shared session - GIL provides atomicity
        if InternalAPIClient._shared_session is None:
            if self.config.debug_api_client_init:
                logger.info("Creating shared HTTP session (GIL-safe singleton pattern)")
            # Create the first base client to establish the session
            self.base_client = BaseAPIClient(self.config)
            # The first client owns the session; reset_singleton() handles final cleanup
            self.base_client._owns_session = False
            # Share the session for reuse (atomic assignment)
            InternalAPIClient._shared_session = self.base_client.session
            InternalAPIClient._shared_base_client = self.base_client
        else:
            if self.config.debug_api_client_init:
                logger.info(
                    f"Reusing shared HTTP session (#{InternalAPIClient._initialization_count})"
                )
            # Create base client and replace its session with shared one
            self.base_client = BaseAPIClient(self.config)
            self.base_client.session.close()  # Close the new session
            self.base_client.session = (
                InternalAPIClient._shared_session
            )  # Use shared session
            self.base_client._owns_session = False

        # Helper to share session on a client
        def _share_session(client: BaseAPIClient) -> None:
            client.session.close()  # Close the freshly-created session
            client.session = InternalAPIClient._shared_session
            client._owns_session = False

        # Create specialized clients with shared session (outside lock for performance)
        self.execution_client = ExecutionAPIClient(self.config)
        _share_session(self.execution_client)

        self.workflow_client = WorkflowAPIClient()
        _share_session(self.workflow_client)

        self.file_client = FileAPIClient(self.config)
        _share_session(self.file_client)

        self.webhook_client = WebhookAPIClient(self.config)
        _share_session(self.webhook_client)

        self.organization_client = OrganizationAPIClient(self.config)
        _share_session(self.organization_client)

        self.tool_client = ToolAPIClient(self.config)
        _share_session(self.tool_client)

        self.usage_client = UsageAPIClient(self.config)
        _share_session(self.usage_client)

    def _initialize_core_clients_traditional(self) -> None:
        """Initialize clients the traditional way (for backward compatibility)."""
        if self.config.debug_api_client_init:
            logger.info(
                "Using traditional API client initialization (8 separate instances)"
            )
        self.base_client = BaseAPIClient(self.config)
        self.execution_client = ExecutionAPIClient(self.config)
        self.workflow_client = WorkflowAPIClient()
        self.file_client = FileAPIClient(self.config)
        self.webhook_client = WebhookAPIClient(self.config)
        self.organization_client = OrganizationAPIClient(self.config)
        self.tool_client = ToolAPIClient(self.config)
        self.usage_client = UsageAPIClient(self.config)

    def _initialize_plugin_clients(self) -> None:
        """Initialize plugin-based clients with error handling."""
        self.manual_review_client = self._load_manual_review_client()
        logger.debug(
            f"Manual review client type: {type(self.manual_review_client).__name__}"
        )

    def _load_manual_review_client(self) -> Any:
        """Load manual review client via plugin registry with fallback.

        Returns:
            ManualReviewClient or ManualReviewNullClient
        """
        try:
            plugin_instance = get_client_plugin("manual_review", self.config)
            if plugin_instance:
                logger.debug("Using manual review plugin from registry")
                return plugin_instance
            else:
                logger.debug("Manual review plugin not available, using null client")
                return ManualReviewNullClient(self.config)
        except Exception as e:
            logger.warning(f"Failed to load manual review plugin, using null client: {e}")
            return ManualReviewNullClient(self.config)

    def _setup_direct_access_references(self) -> None:
        """Setup direct access references for backward compatibility."""
        self.base_url = self.base_client.base_url
        self.api_key = self.base_client.api_key
        self.organization_id = self.base_client.organization_id
        self.session = self.base_client.session

    # Delegate base client methods
    def health_check(self) -> dict[str, Any]:
        """Check API health status."""
        return self.base_client.health_check()

    def close(self):
        """Close API client safely (respects shared session singleton).

        When session sharing is enabled, this method only closes the client
        wrappers but preserves the shared HTTP session for other instances.
        """
        if self.config.enable_api_client_singleton:
            # With session sharing, we don't close individual sessions
            # as they're shared. Only log for debugging.
            if self.config.debug_api_client_init:
                logger.debug(
                    "InternalAPIClient close() called - preserving shared session"
                )
        else:
            # Traditional mode: close all sessions normally
            self.base_client.close()
            self.execution_client.close()
            self.file_client.close()
            self.webhook_client.close()
            self.organization_client.close()
            self.tool_client.close()
            self.workflow_client.close()
            self.usage_client.close()
            logger.debug("Closed all InternalAPIClient sessions (traditional mode)")

    @classmethod
    def reset_singleton(cls):
        """Reset shared singleton session state. Safe to call anytime.

        Note: In thread/gevent/eventlet pools, this may close the session while
        other threads have in-flight requests. The BaseAPIClient retry logic
        handles transient connection errors from this race. For prefork pools,
        this is a non-issue (one task per process).
        """
        if cls._shared_session is not None:
            try:
                cls._shared_session.close()
            except Exception as e:
                logger.warning("Failed to close shared session during reset: %s", e)
            cls._shared_session = None
            cls._shared_base_client = None
            cls._initialization_count = 0
            cls._task_counter = 0
            cls._cached_reset_threshold = None
            logger.info("Reset InternalAPIClient singleton state")

    @classmethod
    def increment_task_counter(cls) -> None:
        """Increment task counter and reset singleton if threshold reached.

        Called via task_postrun signal after each task completes.
        Uses a lock for thread safety in case threads/gevent/eventlet pools are used.
        When singleton disabled (default), reset_singleton() is a no-op.
        """
        if cls._cached_reset_threshold is None:
            cls._cached_reset_threshold = WorkerConfig().singleton_reset_task_threshold

        with cls._task_counter_lock:
            cls._task_counter += 1
            if (
                cls._cached_reset_threshold > 0
                and cls._task_counter >= cls._cached_reset_threshold
            ):
                logger.info(
                    "Task counter reached threshold (%d/%d), resetting singleton session",
                    cls._task_counter,
                    cls._cached_reset_threshold,
                )
                cls.reset_singleton()
                cls._last_reset_time = time.time()

    @classmethod
    def get_task_counter_info(cls) -> dict:
        """Get task counter state for observability."""
        return {
            "task_counter": cls._task_counter,
            "last_reset_time": cls._last_reset_time,
            "shared_session_active": cls._shared_session is not None,
            "initialization_count": cls._initialization_count,
        }

    def __exit__(self, exc_type, exc_val, exc_tb):
        """Context manager exit with safe cleanup."""
        self.close()

        # Close manual review client (plugin or null client)
        if hasattr(self.manual_review_client, "close"):
            self.manual_review_client.close()

    def __enter__(self):
        """Context manager entry."""
        return self

    # Delegate execution client methods
    def get_workflow_executions_by_status(
        self,
        workflow_id: str | uuid.UUID,
        statuses: list[str],
        organization_id: str | None = None,
    ) -> APIResponse:
        """Get workflow executions by status for a specific workflow.

        This gets ALL executions for a workflow that match the given statuses,
        matching the backend logic in source.py _get_active_workflow_executions().

        Args:
            workflow_id: Workflow ID
            statuses: List of status values (e.g., ['PENDING', 'EXECUTING'])
            organization_id: Optional organization ID override

        Returns:
            APIResponse with workflow executions data
        """
        try:
            # Use workflow execution endpoint with status filtering (trailing slash required by Django)
            response_data = self._make_request(
                method=HTTPMethod.GET,
                endpoint="v1/workflow-execution/",
                params={
                    "workflow_id": str(workflow_id),
                    "status__in": ",".join(statuses),  # Multiple status filter
                },
                organization_id=organization_id,
            )

            # Wrap filtered response data in APIResponse
            return APIResponse(
                success=True,
                data=response_data,
            )

        except Exception as e:
            logger.error(f"Error getting workflow executions by status: {e}")
            return APIResponse(
                success=False,
                error=str(e),
            )

    def check_files_active_processing(
        self,
        workflow_id: str | uuid.UUID,
        provider_file_uuids: list[str] = None,
        files: list[dict] = None,
        current_execution_id: str | None = None,
        organization_id: str | None = None,
    ) -> APIResponse:
        """Check if specific files are being processed in PENDING/EXECUTING workflow executions.

        Optimized single API call instead of querying all executions and their file executions.

        Args:
            workflow_id: Workflow ID to check
            provider_file_uuids: [LEGACY] List of provider file UUIDs to check
            files: [NEW] List of file objects with 'uuid' and 'path' fields: [{'uuid': str, 'path': str}]
            current_execution_id: Current execution ID to exclude from active check
            organization_id: Optional organization ID override

        Returns:
            APIResponse with backend response format
        """
        try:
            # Prepare request data with backward compatibility
            request_data = {
                "workflow_id": str(workflow_id),
                "statuses": ["PENDING", "EXECUTING"],
                "current_execution_id": str(current_execution_id)
                if current_execution_id
                else None,
            }

            # Support both legacy and new formats
            if files:
                # New path-aware format
                request_data["files"] = files
            elif provider_file_uuids:
                # Legacy format - fallback
                request_data["provider_file_uuids"] = provider_file_uuids
            else:
                raise ValueError(
                    "Either 'files' or 'provider_file_uuids' must be provided"
                )

            # Single optimized API call to check multiple files at once
            response_data = self._make_request(
                method=HTTPMethod.POST,
                endpoint="v1/workflow-manager/file-execution/check-active",
                data=request_data,
                organization_id=organization_id,
            )

            return APIResponse(
                success=True,
                data=response_data,
            )

        except Exception as e:
            logger.error(f"Error checking files active processing: {e}")
            # Fallback: assume no files are active to avoid blocking
            return APIResponse(
                success=True,
                data={"active_uuids": []},
            )

    def get_workflow_file_executions_by_execution(
        self, execution_id: str | uuid.UUID, organization_id: str | None = None
    ) -> APIResponse:
        """Get WorkflowFileExecutions for a specific execution using internal API.

        This uses the existing /internal/api/v1/file-execution/?execution_id=<id> endpoint
        to get all file executions for a workflow execution.

        Args:
            execution_id: Workflow execution ID
            organization_id: Optional organization ID override

        Returns:
            APIResponse with file executions data
        """
        try:
            response = self._make_request(
                method=HTTPMethod.GET,
                endpoint="v1/file-execution/",
                params={"execution_id": str(execution_id)},
                organization_id=organization_id,
            )

            return response

        except Exception as e:
            logger.error(f"Error getting workflow file executions: {e}")
            return APIResponse(
                success=False,
                error=str(e),
            )

    def get_workflow_execution(
        self,
        execution_id: str | uuid.UUID,
        organization_id: str | None = None,
        include_cost: bool = False,
        file_execution: bool = True,
    ) -> WorkflowExecutionResponse:
        """Get workflow execution with context.

        Args:
            execution_id: Workflow execution ID
            organization_id: Optional organization ID override
            include_cost: Whether to include aggregated usage cost (expensive operation)

        Returns:
            WorkflowExecutionResponse containing workflow execution data
        """
        response = self.execution_client.get_workflow_execution(
            execution_id,
            organization_id,
            include_cost=include_cost,
            file_execution=file_execution,
        )
        # Convert ExecutionResponse to WorkflowExecutionResponse for type safety
        if hasattr(response, "to_dict"):
            response_dict = response.to_dict()
        else:
            response_dict = response
        return WorkflowExecutionResponse.from_api_response(response_dict)

    @with_cache(
        CacheType.WORKFLOW,
        lambda self, workflow_id, organization_id=None: str(workflow_id),
    )
    def get_workflow(
        self, workflow_id: str | uuid.UUID, organization_id: str | None = None
    ) -> WorkflowDefinitionResponseData:
        """Get workflow definition including workflow_type."""
        response = self.workflow_client.get_workflow_definition(
            workflow_id, organization_id
        )
        if not response.success_response:
            raise APIRequestError(response.error_response)
        return response.data

    @with_cache(
        CacheType.WORKFLOW_ENDPOINTS,
        lambda self, workflow_id, organization_id=None: str(workflow_id),
    )
    def get_workflow_endpoints(
        self, workflow_id: str | UUID, organization_id: str | None = None
    ) -> WorkflowEndpointConfigResponseData:
        """Get workflow endpoints for a specific workflow.

        Caching is handled automatically through the @with_cache decorator.
        """
        # Direct API call - caching is handled by decorator
        response = self.workflow_client.get_workflow_endpoints(
            workflow_id, organization_id
        )
        if not response.success_response:
            raise APIRequestError(response.error_response)
        return response.data

    @with_cache(
        CacheType.WORKFLOW_DEFINITION,
        lambda self, workflow_id, organization_id=None: str(workflow_id),
    )
    def get_workflow_definition(
        self, workflow_id: str | uuid.UUID, organization_id: str | None = None
    ) -> WorkflowDefinitionResponse:
        """Get workflow definition including workflow_type.

        This method automatically uses caching through the @with_cache decorator.
        """
        # Direct API call - caching is handled by decorator
        response = self.execution_client.get_workflow_definition(
            workflow_id, organization_id
        )
        # Convert to WorkflowDefinitionResponse for type safety
        if hasattr(response, "to_dict"):
            response_dict = response.to_dict()
        else:
            response_dict = response
        return WorkflowDefinitionResponse.from_api_response(response_dict)

    @with_cache(
        CacheType.PIPELINE,
        lambda self, pipeline_id, organization_id=None: str(pipeline_id),
    )
    def get_pipeline_type(
        self, pipeline_id: str | uuid.UUID, organization_id: str | None = None
    ) -> APIResponse:
        """Get pipeline type by checking APIDeployment and Pipeline models."""
        return self.execution_client.get_pipeline_type(pipeline_id, organization_id)

    @with_cache(
        CacheType.PIPELINE_DATA,
        lambda self, pipeline_id, check_active=True, organization_id=None: (
            f"{pipeline_id}:{check_active}"
        ),
    )
    def get_pipeline_data(
        self,
        pipeline_id: str | uuid.UUID,
        check_active: bool = True,
        organization_id: str | None = None,
    ) -> APIResponse:
        """Get pipeline data by checking APIDeployment and Pipeline models."""
        return self.execution_client.get_pipeline_data(
            pipeline_id=pipeline_id,
            check_active=check_active,
            organization_id=organization_id,
        )

    @with_cache(
        CacheType.API_DEPLOYMENT, lambda self, api_id, organization_id=None: str(api_id)
    )
    def get_api_deployment_data(
        self, api_id: str | uuid.UUID, organization_id: str | None = None
    ) -> APIResponse:
        """Get APIDeployment data directly from v1 API deployment endpoint.

        This method is optimized for callback workers that know they're dealing
        with API deployments. It queries APIDeployment model directly.
        Caching is handled automatically through the @with_cache decorator.
        """
        # Direct API call - caching is handled by decorator
        return self.execution_client.get_api_deployment_data(api_id, organization_id)

    def update_workflow_execution_status(
        self,
        execution_id: str | uuid.UUID,
        status: str,
        error_message: str | None = None,
        total_files: int | None = None,
        successful_files: int | None = None,
        failed_files: int | None = None,
        attempts: int | None = None,
        execution_time: float | None = None,
        organization_id: str | None = None,
        cascade_terminal_files: bool = False,
    ) -> dict[str, Any]:
        """Update workflow execution status.

        ``cascade_terminal_files=True`` also marks the execution's non-terminal file
        executions to the same terminal status atomically (used by the reaper).
        """
        return self.execution_client.update_workflow_execution_status(
            execution_id,
            status,
            error_message,
            total_files,
            successful_files,
            failed_files,
            attempts,
            execution_time,
            organization_id,
            cascade_terminal_files=cascade_terminal_files,
        )

    def recover_stuck_pg_executions(
        self,
        stuck_seconds: int | None = None,
        limit: int | None = None,
    ) -> APIResponse:
        """Reaper safety-net: finalize PG executions stranded non-terminal after all
        their files completed. Org-agnostic (the endpoint scans across orgs, scoped
        to PG rows by ``queue_message_id``). Delegates to the execution client.
        """
        return self.execution_client.recover_stuck_pg_executions(
            stuck_seconds=stuck_seconds, limit=limit
        )

    def create_workflow_execution(self, execution_data: dict[str, Any]) -> dict[str, Any]:
        """Create workflow execution."""
        return self.execution_client.create_workflow_execution(execution_data)

    @with_cache(
        CacheType.TOOL_INSTANCES, lambda self, workflow_id, organization_id: workflow_id
    )
    def get_tool_instances_by_workflow(
        self, workflow_id: str, organization_id: str
    ) -> ToolInstancesResponse:
        """Get tool instances for a workflow.

        Caching is handled automatically through the @with_cache decorator.
        """
        # Direct API call - caching is handled by decorator
        response = self.execution_client.get_tool_instances_by_workflow(
            workflow_id, organization_id
        )
        # Convert to ToolInstancesResponse for type safety
        return ToolInstancesResponse.from_api_response(response)

    def validate_tool_instances(
        self, workflow_id: str, tool_instance_ids: list[str], organization_id: str
    ) -> dict[str, Any]:
        """Validate tool instances and ensure adapter IDs are migrated.

        Args:
            workflow_id: ID of the workflow
            tool_instance_ids: List of tool instance IDs to validate
            organization_id: Organization ID

        Returns:
            Dictionary with validation results:
            {
                "success": bool,
                "validated_instances": [...],  # List of validated instances with migrated metadata
                "errors": [...],  # List of validation errors
                "workflow_id": str
            }
        """
        validation_data = {
            "workflow_id": workflow_id,
            "tool_instances": tool_instance_ids,
        }

        try:
            response = self.execution_client.post(
                "v1/tool-execution/validate/",
                validation_data,
                organization_id=organization_id,
            )
            return response
        except Exception as e:
            logger.error(f"Tool validation API call failed: {e}")
            # Return error response in expected format
            return {
                "success": False,
                "validated_instances": [],
                "errors": [{"error": f"API call failed: {str(e)}"}],
                "workflow_id": workflow_id,
            }

    def compile_workflow(
        self, workflow_id: str, execution_id: str, organization_id: str
    ) -> dict[str, Any]:
        """Compile workflow."""
        return self.execution_client.compile_workflow(
            workflow_id, execution_id, organization_id
        )

    def execute_workflow_async(self, execution_data: dict[str, Any]) -> dict[str, Any]:
        """Execute workflow asynchronously for scheduler.

        This method triggers async workflow execution by dispatching to the
        appropriate Celery task via internal API.

        Args:
            execution_data: Execution parameters including workflow_id, execution_id, etc.

        Returns:
            Dictionary with execution results
        """
        try:
            # Use the internal API to trigger async workflow execution
            # This replaces the Celery send_task("async_execute_bin") call from backend
            response = self.execution_client.post(
                "v1/workflow-manager/execute-async/",
                execution_data,
                organization_id=execution_data.get("organization_id"),
            )

            if isinstance(response, dict) and response.get("success"):
                return {
                    "success": True,
                    "execution_id": response.get("execution_id"),
                    "task_id": response.get("task_id"),
                    "message": "Async workflow execution started",
                }
            else:
                return {
                    "success": False,
                    "error": response.get("error", "Failed to start async execution"),
                }
        except Exception as e:
            logger.error(f"Error starting async workflow execution: {e}")
            return {
                "success": False,
                "error": f"API error: {str(e)}",
            }

    def execute_workflow_async_typed(
        self, async_request: Any
    ) -> SchedulerExecutionResult:
        """Type-safe async workflow execution for scheduler using dataclasses.

        Args:
            async_request: AsyncExecutionRequest dataclass with execution parameters

        Returns:
            SchedulerExecutionResult with execution status and details
        """
        try:
            # Convert dataclass to dict for API call
            execution_data = async_request.to_dict()

            # Use the internal API to trigger async workflow execution
            response = self.execution_client.post(
                "v1/workflow-manager/execute-async/",
                execution_data,
                organization_id=async_request.organization_id,
            )

            if isinstance(response, dict) and response.get("success"):
                return SchedulerExecutionResult.success(
                    execution_id=async_request.execution_id,
                    workflow_id=async_request.workflow_id,
                    pipeline_id=async_request.pipeline_id,
                    task_id=response.get("task_id"),
                    message="Async workflow execution started successfully",
                )
            else:
                return SchedulerExecutionResult.error(
                    error=response.get("error", "Failed to start async execution"),
                    execution_id=async_request.execution_id,
                    workflow_id=async_request.workflow_id,
                    pipeline_id=async_request.pipeline_id,
                )
        except Exception as e:
            logger.error(f"Error starting async workflow execution: {e}")
            return SchedulerExecutionResult.error(
                error=f"API error: {str(e)}",
                execution_id=async_request.execution_id,
                workflow_id=async_request.workflow_id,
                pipeline_id=async_request.pipeline_id,
            )

    def batch_update_execution_status(
        self, updates: list[dict[str, Any]], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Update multiple execution statuses in a single request."""
        # Validate that we have updates to process
        if not updates:
            return {
                "success": True,
                "message": "No updates provided",
                "total_items": 0,
                "successful_items": 0,
                "failed_items": 0,
            }

        result = self.execution_client.batch_update_execution_status(
            updates, organization_id
        )

        # Convert BatchOperationResponse to dict for consistency
        if hasattr(result, "to_dict"):
            response_dict = result.to_dict()
            # Add success field for backward compatibility
            response_dict["success"] = (
                result.status == "SUCCESS" or result.successful_items > 0
            )
            return response_dict
        else:
            return result

    def create_file_batch(
        self,
        workflow_execution_id: str | uuid.UUID,
        files: list,
        is_api: bool = False,
        organization_id: str | None = None,
    ) -> FileBatchResponse:
        """Create file execution batch."""
        response = self.execution_client.create_file_batch(
            workflow_execution_id, files, is_api, organization_id
        )
        # Convert to FileBatchResponse for type safety
        return FileBatchResponse.from_api_response(response)

    def update_pipeline_status(
        self,
        pipeline_id: str | UUID,
        status: str,
        organization_id: str | None = None,
        execution_id: str | UUID | None = None,  # Optional for backward compatibility
        **kwargs,
    ) -> dict[str, Any]:
        """Update pipeline status with flexible parameters."""
        return self.execution_client.update_pipeline_status(
            pipeline_id, status, organization_id, execution_id, **kwargs
        ).data

    def batch_update_pipeline_status(
        self, updates: list[dict[str, Any]], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Update multiple pipeline statuses in a single request.

        For now, this processes updates individually until a batch endpoint is available.
        """
        results = {"success": True, "updated": 0, "errors": []}

        for update in updates:
            try:
                pipeline_id = update.pop("pipeline_id")
                execution_id = update.pop("execution_id")
                status = update.pop("status")

                # Call individual update with remaining kwargs (last_run_time, last_run_status, etc)
                self.update_pipeline_status(
                    pipeline_id=pipeline_id,
                    status=status,
                    organization_id=organization_id,
                    execution_id=execution_id,  # Pass as optional parameter
                    **update,
                )
                results["updated"] += 1
            except Exception as e:
                results["errors"].append({"pipeline_id": pipeline_id, "error": str(e)})
                results["success"] = False

        return results

    # Workflow execution finalization handled by status updates

    def cleanup_execution_resources(
        self, execution_ids: list[str | uuid.UUID], cleanup_types: list | None = None
    ) -> dict[str, Any]:
        """Cleanup execution resources."""
        return self.execution_client.cleanup_execution_resources(
            execution_ids, cleanup_types
        )

    # get_execution_finalization_status method removed - it was dead code
    # Only used by finalize_execution_callback which was never called

    def increment_completed_files(
        self, workflow_id: str, execution_id: str
    ) -> dict[str, Any]:
        """Increment completed files count for execution."""
        return self.execution_client.increment_completed_files(workflow_id, execution_id)

    def increment_failed_files(
        self, workflow_id: str, execution_id: str
    ) -> dict[str, Any]:
        """Increment failed files count for execution."""
        return self.execution_client.increment_failed_files(workflow_id, execution_id)

    def get_workflow_destination_config(
        self, workflow_id: str, execution_id: str
    ) -> dict[str, Any]:
        """Get destination configuration for workflow execution."""
        return self.execution_client.get_workflow_destination_config(
            workflow_id, execution_id
        )

    # Delegate file client methods
    def get_workflow_file_execution(
        self, file_execution_id: str | UUID, organization_id: str | None = None
    ) -> WorkflowFileExecutionData:
        """Get an existing workflow file execution by ID."""
        return self.file_client.get_workflow_file_execution(
            file_execution_id, organization_id
        )

    def get_or_create_workflow_file_execution(
        self,
        execution_id: str | UUID,
        file_hash: dict[str, Any] | FileHashData,
        workflow_id: str | UUID,
        organization_id: str | None = None,
        force_create: bool = False,
    ) -> WorkflowFileExecutionData:
        """Get or create a workflow file execution record using shared dataclasses."""
        return self.file_client.get_or_create_workflow_file_execution(
            execution_id, file_hash, workflow_id, organization_id, force_create
        )

    def update_workflow_file_execution_hash(
        self,
        file_execution_id: str | UUID,
        file_hash: str,
        fs_metadata: dict[str, Any] | None = None,
        mime_type: str | None = None,
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Update workflow file execution with computed file hash and mime_type."""
        return self.file_client.update_workflow_file_execution_hash(
            file_execution_id, file_hash, fs_metadata, mime_type, organization_id
        )

    def update_file_execution_status(
        self,
        file_execution_id: str | UUID,
        status: str,
        execution_time: float | None = None,
        error_message: str | None = None,
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Update workflow file execution status with execution time."""
        return self.file_client.update_file_execution_status(
            file_execution_id, status, execution_time, error_message, organization_id
        )

    def update_workflow_file_execution_status(
        self,
        file_execution_id: str,
        status: str,
        result: str | None = None,
        error_message: str | None = None,
    ) -> dict[str, Any]:
        """Update WorkflowFileExecution status via internal API using shared dataclasses."""
        return self.file_client.update_workflow_file_execution_status(
            file_execution_id, status, result, error_message
        )

    def update_file_status_to_executing(
        self,
        file_execution_id: str | UUID | None,
        file_name: str,
        organization_id: str | None = None,
    ) -> bool:
        """Common method to update file execution status to EXECUTING with proper error handling.

        This method provides consistent logging and error handling for updating file execution
        status to EXECUTING across all workflow types (ETL, TASK, API).

        Args:
            file_execution_id: File execution ID to update
            file_name: File name for logging purposes
            organization_id: Optional organization ID override

        Returns:
            bool: True if update successful, False otherwise
        """
        if not file_execution_id:
            return False

        try:
            result = self.update_file_execution_status(
                file_execution_id=file_execution_id,
                status=ExecutionStatus.EXECUTING.value,
                organization_id=organization_id,
            )
            logger.info(
                f"Updated file {file_name} (ID: {file_execution_id}) status to EXECUTING :{result}"
            )
            return True

        except Exception as status_error:
            logger.error(
                f"CRITICAL: Failed to update file {file_name} (ID: {file_execution_id}) status to EXECUTING: {status_error}"
            )
            import traceback

            logger.error(f"Traceback: {traceback.format_exc()}")
            return False

    def create_workflow_file_execution(
        self,
        workflow_execution_id: str,
        file_name: str,
        file_path: str,
        file_hash: str,
        file_execution_id: str | None = None,
        file_size: int = 0,
        mime_type: str = "",
        provider_file_uuid: str | None = None,
        fs_metadata: dict[str, Any] | None = None,
        status: str | ExecutionStatus = ExecutionStatus.PENDING,
    ) -> dict[str, Any]:
        """Create WorkflowFileExecution record via internal API with complete metadata."""
        return self.file_client.create_workflow_file_execution(
            workflow_execution_id,
            file_name,
            file_path,
            file_hash,
            file_execution_id,
            file_size,
            mime_type,
            provider_file_uuid,
            fs_metadata,
            status,
        )

    def batch_create_file_executions(
        self, file_executions: list[dict[str, Any]], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Create multiple file executions in a single batch request."""
        return self.file_client.batch_create_file_executions(
            file_executions, organization_id
        )

    def batch_update_file_execution_status(
        self, status_updates: list[dict[str, Any]], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Update multiple file execution statuses in a single batch request."""
        return self.file_client.batch_update_file_execution_status(
            status_updates, organization_id
        )

    def get_file_history_by_cache_key(
        self, cache_key: str, workflow_id: str | uuid.UUID, file_path: str | None = None
    ) -> dict[str, Any]:
        """Get file history by cache key."""
        return self.file_client.get_file_history_by_cache_key(
            cache_key, workflow_id, file_path
        )

    def reserve_file_processing(
        self,
        workflow_id: str | uuid.UUID,
        cache_key: str,
        provider_file_uuid: str | None = None,
        file_path: str | None = None,
        worker_id: str | None = None,
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Atomic check-and-reserve operation for file processing deduplication."""
        result = self.file_client.reserve_file_processing(
            workflow_id,
            cache_key,
            provider_file_uuid,
            file_path,
            worker_id,
            organization_id,
        )
        return (
            result.data if result.success else {"reserved": False, "error": result.error}
        )

    def create_file_history(
        self,
        file_path: str,
        file_name: str,
        source_connection_type: str,
        workflow_id: str | uuid.UUID,
        result: str | None = None,
        metadata: dict[str, Any] | None = None,
        status: str = "COMPLETED",
        error: str | None = None,
        provider_file_uuid: str | None = None,
        is_api: bool = False,
        file_size: int = 0,
        file_hash: str = "",
        mime_type: str = "",
    ) -> dict[str, Any]:
        """Create file history record matching backend expected format."""
        return self.file_client.create_file_history(
            file_path=file_path,
            file_name=file_name,
            source_connection_type=source_connection_type,
            workflow_id=workflow_id,
            result=result,
            metadata=metadata,
            status=status,
            error=error,
            provider_file_uuid=provider_file_uuid,
            is_api=is_api,
            file_size=file_size,
            file_hash=file_hash,
            mime_type=mime_type,
        )

    def get_file_history_status(self, file_history_id: str | uuid.UUID) -> dict[str, Any]:
        """Get file history status."""
        return self.file_client.get_file_history_status(file_history_id)

    def get_file_history(
        self,
        workflow_id: str | uuid.UUID,
        provider_file_uuid: str | None = None,
        file_hash: str | None = None,
        file_path: str | None = None,
        organization_id: str | None = None,
    ) -> FileHistoryResponse:
        """Get file history by provider_file_uuid or file_hash.

        This unified method handles both lookup patterns:
        1. By provider_file_uuid (for cloud storage files with unique IDs)
        2. By file_hash (content-based deduplication)

        Args:
            workflow_id: Workflow ID to check file history for
            provider_file_uuid: Provider file UUID to search for (cloud storage)
            file_hash: File content hash to search for (content-based)
            file_path: Optional file path for additional filtering
            organization_id: Organization ID for context

        Returns:
            FileHistoryResponse with file_history data
        """
        # If file_hash is provided, use the existing cache key lookup
        if file_hash and not provider_file_uuid:
            response = self.get_file_history_by_cache_key(
                cache_key=file_hash, workflow_id=workflow_id, file_path=file_path
            )
            return FileHistoryResponse.from_api_response(response)

        # Otherwise, use provider_file_uuid lookup
        endpoint = "v1/file-history/get/"

        payload = {
            "workflow_id": str(workflow_id),
            "provider_file_uuid": provider_file_uuid,
            "file_path": file_path,
            "organization_id": organization_id or self.organization_id,
        }

        try:
            response = self.base_client._make_request(
                method="POST",
                endpoint=endpoint,
                data=payload,
                timeout=self.base_client.config.api_timeout,
                organization_id=organization_id,
            )

            logger.debug(
                f"File history lookup for provider_file_uuid {provider_file_uuid}: "
                f"{'found' if response.get('file_history') else 'not found'}"
            )
            return FileHistoryResponse.from_api_response(response)

        except Exception as e:
            logger.error(
                f"Failed to get file history for workflow {workflow_id}, "
                f"provider_file_uuid {provider_file_uuid}: {str(e)}"
            )
            # Return empty result to continue without breaking the flow
            return FileHistoryResponse(success=False, error=str(e))

    def batch_create_file_history(
        self, file_histories: list[dict[str, Any]], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Create multiple file history records in a single batch request."""
        return self.file_client.batch_create_file_history(file_histories, organization_id)

    def get_file_history_flexible(
        self,
        workflow_id: str | uuid.UUID,
        cache_key: str | None = None,
        provider_file_uuid: str | None = None,
        file_path: str | None = None,
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Get file history using flexible parameters (cache_key OR provider_file_uuid)."""
        return self.file_client.get_file_history_flexible(
            workflow_id=workflow_id,
            cache_key=cache_key,
            provider_file_uuid=provider_file_uuid,
            file_path=file_path,
            organization_id=organization_id,
        )

    def get_files_history_batch(
        self,
        workflow_id: str | uuid.UUID,
        files: list[dict[str, str]],
        organization_id: str | None = None,
    ) -> dict[str, dict[str, Any]]:
        """Get file history for multiple files in a single batch operation."""
        return self.file_client.get_files_history_batch(
            workflow_id=workflow_id,
            files=files,
            organization_id=organization_id,
        )

    # Delegate webhook client methods
    def send_webhook(
        self,
        url: str,
        payload: dict[str, Any],
        notification_id: str | None = None,
        authorization_type: str = "NONE",
        authorization_key: str | None = None,
        authorization_header: str | None = None,
        timeout: int = 30,
        max_retries: int = 3,
        retry_delay: int = 5,
        headers: dict[str, str] | None = None,
    ) -> dict[str, Any]:
        """Send webhook notification to external endpoint."""
        return self.webhook_client.send_webhook(
            url,
            payload,
            notification_id,
            authorization_type,
            authorization_key,
            authorization_header,
            timeout,
            max_retries,
            retry_delay,
            headers,
        )

    def send_webhook_notification(
        self,
        notification_request: WebhookNotificationRequest,
    ) -> dict[str, Any]:
        """Send webhook notification using type-safe NotificationRequest dataclass.

        Args:
            notification_request: WebhookNotificationRequest dataclass with notification details

        Returns:
            Dictionary with webhook response data
        """
        return self.webhook_client.send_webhook(
            url=notification_request.url,
            payload=notification_request.payload,
            notification_id=notification_request.notification_id,
            authorization_type=notification_request.authorization_type,
            authorization_key=notification_request.authorization_key,
            authorization_header=notification_request.authorization_header,
            timeout=notification_request.timeout,
            max_retries=notification_request.max_retries,
            retry_delay=notification_request.retry_delay,
            headers=notification_request.headers,
        )

    def get_webhook_status(self, task_id: str) -> dict[str, Any]:
        """Get webhook delivery status by task ID."""
        return self.webhook_client.get_webhook_status(task_id)

    def test_webhook(
        self,
        url: str,
        payload: dict[str, Any],
        authorization_type: str = "NONE",
        authorization_key: str | None = None,
        authorization_header: str | None = None,
        timeout: int = 10,
    ) -> dict[str, Any]:
        """Test webhook configuration and connectivity."""
        return self.webhook_client.test_webhook(
            url,
            payload,
            authorization_type,
            authorization_key,
            authorization_header,
            timeout,
        )

    # Delegate tool client methods (removing duplicates)
    def get_tool_instances_by_workflow_tool_client(
        self, workflow_id: str | uuid.UUID
    ) -> dict[str, Any]:
        """Get tool instances for a workflow using tool client."""
        # Use the tool client for consistency
        result = self.tool_client.get_tool_instances_by_workflow(workflow_id)
        return result.data if hasattr(result, "data") else result

    # File history methods
    def check_file_history_batch(
        self,
        workflow_id: str | uuid.UUID,
        file_hashes: list[str],
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Check file history for a batch of file hashes.

        Args:
            workflow_id: Workflow ID to check file history for
            file_hashes: List of file hashes to check
            organization_id: Organization ID for context (optional)

        Returns:
            Dictionary with 'processed_file_hashes' list
        """
        endpoint = "v1/workflow-manager/file-history/check-batch/"

        payload = {
            "workflow_id": str(workflow_id),
            "file_hashes": file_hashes,
            "organization_id": organization_id or self.organization_id,
        }

        try:
            response = self.base_client._make_request(
                method="POST",
                endpoint=endpoint,
                data=payload,
                timeout=self.base_client.config.api_timeout,
                organization_id=organization_id,
            )

            logger.debug(
                f"File history batch check for {len(file_hashes)} hashes: {len(response.get('processed_file_hashes', []))} already processed"
            )
            return response

        except Exception as e:
            logger.error(
                f"Failed to check file history batch for workflow {workflow_id}: {str(e)}"
            )
            # Return empty result to continue without file history filtering
            return {"processed_file_hashes": []}

    # Delegate organization client methods
    def get_organization_context(self, org_id: str) -> dict[str, Any]:
        """Get organization context and metadata."""
        return self.organization_client.get_organization_context(org_id)

    def set_organization_context(self, org_id: str):
        """Set organization context for subsequent requests."""
        # Update context on all clients
        self.base_client.set_organization_context(org_id)
        self.execution_client.set_organization_context(org_id)
        self.file_client.set_organization_context(org_id)
        self.webhook_client.set_organization_context(org_id)
        self.organization_client.set_organization_context(org_id)
        self.tool_client.set_organization_context(org_id)
        self.workflow_client.set_organization_context(org_id)
        self.usage_client.set_organization_context(org_id)

        # Note: Manual review org context handled by plugins

        # Update facade attributes
        self.organization_id = org_id

        logger.debug(f"Set organization context to {org_id} on all clients")

    def clear_organization_context(self):
        """Clear organization context."""
        # Clear context on all clients
        self.base_client.clear_organization_context()
        self.execution_client.clear_organization_context()
        self.file_client.clear_organization_context()
        self.webhook_client.clear_organization_context()
        self.organization_client.clear_organization_context()
        self.tool_client.clear_organization_context()
        self.workflow_client.clear_organization_context()
        self.usage_client.clear_organization_context()

        # Note: Manual review org context clearing handled by plugins

        # Update facade attributes
        self.organization_id = None

        logger.debug("Cleared organization context on all clients")

    # Manual review functionality is handled by enterprise plugins, not exposed in OSS API client

    # Delegate tool client methods (consolidated)
    def execute_tool(
        self,
        tool_instance_id: str | uuid.UUID,
        input_data: dict[str, Any],
        file_data: dict[str, Any] | None = None,
        execution_context: dict[str, Any] | None = None,
    ) -> dict[str, Any]:
        """Execute a tool instance with provided data."""
        result = self.tool_client.execute_tool(
            tool_instance_id, input_data, file_data, execution_context
        )
        return result.data if hasattr(result, "data") else result

    def get_tool_execution_status(self, execution_id: str | uuid.UUID) -> dict[str, Any]:
        """Get tool execution status by execution ID."""
        result = self.tool_client.get_tool_execution_status(execution_id)
        return result.data if hasattr(result, "data") else result

    def get_tool_by_id(self, tool_id: str | uuid.UUID) -> dict[str, Any]:
        """Get tool information by tool ID."""
        return self.base_client.get(
            self.base_client._build_url("tool_execution", f"tool/{str(tool_id)}/")
        )

    # HTTP method helpers (delegate to base client)
    def get(
        self,
        endpoint: str,
        params: dict[str, Any] | None = None,
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Make GET request."""
        return self.base_client.get(endpoint, params, organization_id)

    def sweep_undispatched_executions(self) -> dict[str, Any]:
        """Ask the backend to terminalise executions that were never dispatched.

        Closes a gap no worker can see: an abort between ``create_workflow_execution``
        and ``execute_workflow_async`` commits a PENDING execution row without ever
        creating a queue message or a barrier, so the reaper's barrier scan cannot
        find it and PENDING — a non-terminal state — is never resolved.

        Backend-side because the sweep touches ``WorkflowExecution``, the API-deployment
        rate limiter; the reaper only supplies leader
        election so exactly one instance runs it. Org-agnostic: the sweep spans all
        organizations, so no ``organization_id`` is sent.
        """
        return self.post("v1/execution-logs/sweep-undispatched-executions/", data={})

    def post(
        self, endpoint: str, data: dict[str, Any], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Make POST request."""
        return self.base_client.post(endpoint, data, organization_id)

    def put(
        self, endpoint: str, data: dict[str, Any], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Make PUT request."""
        return self.base_client.put(endpoint, data, organization_id)

    def patch(
        self, endpoint: str, data: dict[str, Any], organization_id: str | None = None
    ) -> dict[str, Any]:
        """Make PATCH request."""
        return self.base_client.patch(endpoint, data, organization_id)

    def delete(self, endpoint: str, organization_id: str | None = None) -> dict[str, Any]:
        """Make DELETE request."""
        return self.base_client.delete(endpoint, organization_id)

    # Private helpers for internal access
    def _build_url(self, endpoint_key: str, path: str = "") -> str:
        """Build consistent API URL using endpoint patterns."""
        return self.base_client._build_url(endpoint_key, path)

    def _make_request(
        self,
        method: str,
        endpoint: str,
        data: dict[str, Any] | None = None,
        params: dict[str, Any] | None = None,
        timeout: int | None = None,
        max_retries: int = 3,
        backoff_factor: float = 0.5,
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Enhanced HTTP request with robust error handling and retry logic."""
        return self.base_client._make_request(
            method,
            endpoint,
            data,
            params,
            timeout,
            max_retries,
            backoff_factor,
            organization_id,
        )

    def get_endpoint_config(self) -> dict[str, str]:
        """Get current API endpoint configuration for debugging."""
        return self.base_client.get_endpoint_config()

    # Manual Review API methods (delegated to ManualReviewAPIClient)
    def validate_manual_review_db_rule(
        self,
        execution_result: Any,
        workflow_id: str | UUID,
        file_destination: str | None = None,
        organization_id: str | None = None,
    ) -> bool:
        """Validate if document should go to manual review based on DB rules."""
        return self.manual_review_client.validate_manual_review_db_rule(
            execution_result=execution_result,
            workflow_id=workflow_id,
            file_destination=file_destination,
            organization_id=organization_id,
        )

    def enqueue_manual_review(
        self,
        queue_name: str,
        message: dict[str, Any],
        organization_id: str | None = None,
    ) -> bool:
        """Enqueue document for manual review."""
        return self.manual_review_client.enqueue_manual_review(
            queue_name=queue_name,
            message=message,
            organization_id=organization_id,
        )

    def route_to_manual_review(
        self,
        file_execution_id: str,
        file_data: dict[str, Any],
        workflow_id: str,
        execution_id: str,
        organization_id: str | None = None,
    ) -> dict[str, Any]:
        """Route file to manual review queue (delegates to manual review client)."""
        return self.manual_review_client.route_to_manual_review(
            file_execution_id=file_execution_id,
            file_data=file_data,
            workflow_id=workflow_id,
            execution_id=execution_id,
            organization_id=organization_id or self.organization_id,
        )

    def route_to_manual_review_with_results(
        self,
        file_execution_id: str,
        file_data: dict[str, Any],
        workflow_result: dict[str, Any],
        workflow_id: str,
        execution_id: str,
        organization_id: str | None = None,
        file_name: str = "unknown",
    ) -> dict[str, Any]:
        """Route file to manual review with tool execution results (delegates to manual review client)."""
        return self.manual_review_client.route_with_results(
            file_execution_id=file_execution_id,
            file_data=file_data,
            workflow_result=workflow_result,
            workflow_id=workflow_id,
            execution_id=execution_id,
            organization_id=organization_id or self.organization_id,
            file_name=file_name,
        )

    # Configuration client methods
    def get_configuration(
        self,
        config_key: str,
        organization_id: str | None = None,
    ) -> "APIResponse":
        """Get organization configuration value.

        Args:
            config_key: Configuration key name (e.g., "MAX_PARALLEL_FILE_BATCHES")
            organization_id: Organization ID (uses client default if not provided)

        Returns:
            APIResponse with configuration data
        """
        try:
            response = self._make_request(
                method=HTTPMethod.GET,
                endpoint=f"v1/configuration/{config_key}/",
                params={"organization_id": organization_id or self.organization_id},
            )

            return response

        except Exception as e:
            logger.error(f"Error getting configuration {config_key}: {e}")
            from .data.models import APIResponse

            return APIResponse(
                success=False,
                error=str(e),
            )

    # Agent-KV client methods (spec §5.3/§5.4)
    def agent_kv_finalize(
        self,
        job_id: str,
        org_id: str,
        success: bool,
        result: dict[str, Any] | None = None,
        error: str = "",
        usage_summary: dict[str, Any] | None = None,
    ) -> dict[str, Any]:
        """Terminalize an Agent-KV job via the internal finalize endpoint (Task 11).

        Idempotent server-side: a job already in a terminal state short-circuits
        to ``{"finalized": False, ...}`` instead of rewriting its result.

        Args:
            job_id: Agent-KV job ID.
            org_id: Organization ID (required by the endpoint's body, not the
                URL — mirrors ``StageReportView``/``FinalizeView``).
            success: Whether the executor run succeeded. Must be a real bool —
                the endpoint 400s on a non-bool value.
            result: Engine result to persist as the job's output (success only).
            error: Failure reason to persist (failure only).
            usage_summary: Optional usage/cost summary to persist (success only).

        Returns:
            Backend response: ``{"finalized": bool, "webhook_url": str,
            "status": str, "reason": str, "result_attached": bool}``.
            ``result_attached`` is True only when ``finalized`` is False because
            a cancel won the race, and the finished extraction's result was kept
            on the cancelled row rather than discarded -- the one non-fresh
            finalize that still owes the caller a notification (see
            ``ide_callback.agent_kv_tasks._maybe_webhook``).
        """
        payload: dict[str, Any] = {"org_id": org_id, "success": success}
        if success:
            payload["result"] = result or {}
            if usage_summary is not None:
                payload["usage_summary"] = usage_summary
        else:
            payload["error"] = error
        return self.post(
            f"v1/agent-kv/jobs/{job_id}/finalize/",
            data=payload,
            organization_id=org_id,
        )

    def agent_kv_stage_report(
        self,
        job_id: str,
        org_id: str,
        stage: str,
        status: str,
        seconds: float | None = None,
        counters: dict[str, Any] | None = None,
    ) -> dict[str, Any]:
        """Report a pipeline stage for an Agent-KV job via the internal
        stage-report endpoint (spec §5.3/§5.4).

        Idempotent server-side, mirroring ``agent_kv_finalize``: a job already
        in a terminal state short-circuits to ``{"ok": True, "noop": True}``
        instead of recording the stage.

        Args:
            job_id: Agent-KV job ID.
            org_id: Organization ID (required by the endpoint's body, not the
                URL — mirrors ``StageReportView``/``FinalizeView``).
            stage: Pipeline stage name. Must be one of the stage names the
                job's extractor declares (``agent_kv.constants
                .STAGE_NAMES_BY_EXTRACTOR``) -- ``"table_extraction"`` for
                ``table``, one of ``STAGE_NAMES`` for ``kv``. The endpoint
                only checks the name is non-empty, but ``_status_document``
                filters a job's recorded stages through that list, so an
                off-list name is stored and then dropped from every status
                response.
            status: Stage status -- ``"running"`` or ``"done"``, the only two
                values ``StageReportView`` accepts. (These examples read
                ``"started"``/``"completed"`` before review; both are 400s.)
            seconds: Optional stage duration in seconds.
            counters: Optional stage counters to record.

        Returns:
            Backend response: ``{"ok": True}`` or ``{"ok": True, "noop": True}``.
        """
        payload: dict[str, Any] = {"org_id": org_id, "stage": stage, "status": status}
        if seconds is not None:
            payload["seconds"] = seconds
        if counters:
            payload["counters"] = counters
        return self.post(
            f"v1/agent-kv/jobs/{job_id}/stage/",
            data=payload,
            organization_id=org_id,
        )

    def agent_kv_sweep(self) -> dict[str, Any]:
        """Trigger the Agent-KV maintenance sweep via the internal endpoint
        (spec §5.4, Fix 5/Fix 8).

        Platform-wide, like the endpoint itself (``SweepView``): no
        ``org_id`` -- it terminalizes never-dispatched ``PENDING`` jobs and
        stuck ``DISPATCHED``/``RUNNING`` jobs across every org in one call.
        Idempotent and batch-capped server-side, so safe to call on a tight
        schedule.

        Returns:
            Backend response: ``{"swept": int, "timed_out": int}``.
        """
        return self.post("v1/agent-kv/sweep/", data={})

    def agent_kv_ttl_cleanup(self) -> dict[str, Any]:
        """Trigger the Agent-KV TTL cleanup via the internal endpoint
        (spec §5.4, Fix 5).

        Platform-wide, like the endpoint itself (``TTLCleanupView``): no
        ``org_id`` -- it deletes staged input/result files for jobs past
        ``expires_at`` and blanks their refs, across every org in one call.
        Idempotent and batch-capped server-side.

        Returns:
            Backend response: ``{"cleaned": int}``.
        """
        return self.post("v1/agent-kv/ttl-cleanup/", data={})

    # Usage client methods (delegate to UsageAPIClient)
    def get_aggregated_token_count(
        self, file_execution_id: str, organization_id: str | None = None
    ) -> UsageResponseData | None:
        """Get aggregated token usage data for a file execution (backward compatibility wrapper).

        Args:
            file_execution_id: File execution ID to get usage data for
            organization_id: Optional organization ID override

        Returns:
            Dictionary with aggregated usage data for backward compatibility
        """
        response = self.usage_client.get_aggregated_token_count(
            file_execution_id, organization_id
        )

        if response.success and response.data:
            # Return dict format for backward compatibility with existing code
            return response.data
        else:
            logger.warning(
                f"No usage data found for file_execution_id {file_execution_id}"
            )
            return None

    def get_aggregated_pages_processed(
        self, file_execution_id: str, organization_id: str | None = None
    ) -> int | None:
        """Get aggregated pages processed for a file execution.

        Args:
            file_execution_id: File execution ID to get pages data for
            organization_id: Optional organization ID override

        Returns:
            Total pages processed count, or None if no data found
        """
        return self.usage_client.get_aggregated_pages_processed(
            file_execution_id, organization_id
        )

    # Export all classes and exceptions for backward compatibility
    # =============================
    # CACHE MANAGEMENT UTILITIES
    # =============================
    # These methods provide cache management without overriding existing methods
    # The caching happens transparently in the execution client methods

    def get_api_cache_stats(self) -> dict[str, Any]:
        """Get API client cache statistics."""
        if hasattr(self, "_cache") and self._cache.backend.available:
            return self.get_cache_stats()
        return {"available": False, "message": "Cache not available"}

    def clear_api_cache_stats(self):
        """Clear API client cache statistics."""
        if hasattr(self, "_cache") and self._cache.backend.available:
            self.clear_cache_stats()

    def invalidate_workflow_related_cache(self, workflow_id: str):
        """Invalidate all cache entries related to a workflow."""
        if hasattr(self, "_cache") and self._cache.backend.available:
            self.invalidate_workflow_cache(workflow_id)
            logger.info(f"Invalidated all cache entries for workflow {workflow_id}")
        else:
            logger.debug(
                f"Cache not available, skipping invalidation for workflow {workflow_id}"
            )


__all__ = [
    # Main facade class
    "InternalAPIClient",
    # Exceptions
    "InternalAPIClientError",
    "APIRequestError",
    "AuthenticationError",
    # Data models
    "WorkflowFileExecutionData",
    "FileHashData",
    "FileExecutionCreateRequest",
    "FileExecutionStatusUpdateRequest",
]