LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3248 [FIX] Add total_pages_processed to worker destination metadata (#1833)
Praveen Kumare1d88bebbf4b
diff --git a/backend/usage_v2/internal_urls.py b/backend/usage_v2/internal_urls.pyindex a3a02ba7a..b5a867555 100644--- a/backend/usage_v2/internal_urls.py+++ b/backend/usage_v2/internal_urls.py@@ -12,4 +12,9 @@ urlpatterns = [internal_views.UsageInternalView.as_view(),name="aggregated-token-count",),+ path(+ "aggregated-pages-processed/<str:file_execution_id>/",+ internal_views.PagesProcessedInternalView.as_view(),+ name="aggregated-pages-processed",+ ),]diff --git a/backend/usage_v2/internal_views.py b/backend/usage_v2/internal_views.pyindex 864ae6ae9..3c28779c5 100644--- a/backend/usage_v2/internal_views.py+++ b/backend/usage_v2/internal_views.py@@ -77,3 +77,59 @@ class UsageInternalView(APIView):},status=status.HTTP_500_INTERNAL_SERVER_ERROR,)+++class PagesProcessedInternalView(APIView):+ """Internal API view for workers to access aggregated pages processed.++ This endpoint allows workers to get aggregated pages processed+ for a specific file execution without direct database access.+ """++ def get(self, request: Request, file_execution_id: str) -> JsonResponse:+ """Get aggregated pages processed for a file execution.++ Args:+ request: HTTP request+ file_execution_id: File execution ID to get pages data for++ Returns:+ JSON response with aggregated pages processed count+ """+ try:+ if not file_execution_id:+ return JsonResponse(+ {+ "success": False,+ "error": "file_execution_id parameter is required",+ },+ status=status.HTTP_400_BAD_REQUEST,+ )++ total_pages = UsageHelper.get_aggregated_pages_processed(+ run_id=file_execution_id+ )++ return JsonResponse(+ {+ "success": True,+ "data": {+ "file_execution_id": file_execution_id,+ "total_pages_processed": total_pages,+ },+ }+ )++ except Exception as e:+ logger.error(+ f"Error getting pages processed for file_execution_id {file_execution_id}: {e}",+ exc_info=True,+ )+ return JsonResponse(+ {+ "success": False,+ "error": "Internal server error",+ "file_execution_id": file_execution_id,+ },+ status=status.HTTP_500_INTERNAL_SERVER_ERROR,+ )diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.pyindex a06605754..dc19db881 100644--- a/workers/shared/api/internal_client.py+++ b/workers/shared/api/internal_client.py@@ -1536,6 +1536,22 @@ class InternalAPIClient(CachedAPIClientMixin):)return None+ def get_aggregated_pages_processed(+ self, file_execution_id: str, organization_id: str | None = None+ ) -> int | None:+ """Get aggregated pages processed for a file execution.++ Args:+ file_execution_id: File execution ID to get pages data for+ organization_id: Optional organization ID override++ Returns:+ Total pages processed count, or None if no data found+ """+ return self.usage_client.get_aggregated_pages_processed(+ file_execution_id, organization_id+ )+# Export all classes and exceptions for backward compatibility# =============================# CACHE MANAGEMENT UTILITIESdiff --git a/workers/shared/clients/usage_client.py b/workers/shared/clients/usage_client.pyindex 39c30fcc9..a7ba8c701 100644--- a/workers/shared/clients/usage_client.py+++ b/workers/shared/clients/usage_client.py@@ -170,3 +170,30 @@ class UsageAPIClient(BaseAPIClient, UsageOperationMixin):file_execution_id=validated_file_execution_id,message="Failed to retrieve usage data",)++ def get_aggregated_pages_processed(+ self, file_execution_id: str | uuid.UUID, organization_id: str | None = None+ ) -> int | None:+ """Get aggregated pages processed for a file execution.++ Args:+ file_execution_id: File execution ID to get pages data for+ organization_id: Optional organization ID override++ Returns:+ Total pages processed count, or None if no data found+ """+ try:+ validated_id = self._validate_file_execution_id(file_execution_id)+ endpoint = f"v1/usage/aggregated-pages-processed/{validated_id}/"+ response = self.get(endpoint, organization_id=organization_id)++ if response and response.get("success"):+ return response.get("data", {}).get("total_pages_processed")+ return None++ except Exception as e:+ logger.error(+ f"Failed to get pages processed for {file_execution_id}: {str(e)}"+ )+ return Nonediff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.pyindex 9207556db..7747ae5db 100644--- a/workers/shared/workflow/destination_connector.py+++ b/workers/shared/workflow/destination_connector.py@@ -629,8 +629,9 @@ class WorkerDestinationConnector:f"🔌 File '{file_ctx.file_name}' marked for API processing - preparing response",)- # Prepare metadata - add HITL info only if plugin is available (cloud feature)- api_metadata = result.metadata.copy() if result.metadata else {}+ # Enrich metadata with usage and pages processed data+ api_metadata = self.get_combined_metadata(exec_ctx.api_client, result.metadata)+ # Add HITL info only if plugin is available (cloud feature)if has_manual_review_plugin():api_metadata["hitl"] = {"file_sent_to_hitl": result.has_hitl,@@ -900,11 +901,19 @@ class WorkerDestinationConnector:Returns:dict[str, Any]: Combined metadata including workflow and usage data."""+ if metadata is None:+ metadata = {}+ if not api_client:+ return metadatafile_execution_id = self.file_execution_idusage_metadata = api_client.get_aggregated_token_count(file_execution_id)- if metadata and usage_metadata:+ if usage_metadata:metadata["usage"] = usage_metadata.to_dict()+ if file_execution_id:+ metadata["total_pages_processed"] = api_client.get_aggregated_pages_processed(+ file_execution_id+ )return metadatadef insert_into_db(