Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-3248 [FIX] Add total_pages_processed to worker destination metadata (#1833)

Praveen Kumare1d88bebbf4b
diff --git a/backend/usage_v2/internal_urls.py b/backend/usage_v2/internal_urls.py
index a3a02ba7a..b5a867555 100644
--- a/backend/usage_v2/internal_urls.py
+++ b/backend/usage_v2/internal_urls.py
@@ -12,4 +12,9 @@ urlpatterns = [
internal_views.UsageInternalView.as_view(),
name="aggregated-token-count",
),
+ path(
+ "aggregated-pages-processed/<str:file_execution_id>/",
+ internal_views.PagesProcessedInternalView.as_view(),
+ name="aggregated-pages-processed",
+ ),
]
diff --git a/backend/usage_v2/internal_views.py b/backend/usage_v2/internal_views.py
index 864ae6ae9..3c28779c5 100644
--- a/backend/usage_v2/internal_views.py
+++ b/backend/usage_v2/internal_views.py
@@ -77,3 +77,59 @@ class UsageInternalView(APIView):
},
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
+
+
+class PagesProcessedInternalView(APIView):
+ """Internal API view for workers to access aggregated pages processed.
+
+ This endpoint allows workers to get aggregated pages processed
+ for a specific file execution without direct database access.
+ """
+
+ def get(self, request: Request, file_execution_id: str) -> JsonResponse:
+ """Get aggregated pages processed for a file execution.
+
+ Args:
+ request: HTTP request
+ file_execution_id: File execution ID to get pages data for
+
+ Returns:
+ JSON response with aggregated pages processed count
+ """
+ try:
+ if not file_execution_id:
+ return JsonResponse(
+ {
+ "success": False,
+ "error": "file_execution_id parameter is required",
+ },
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+
+ total_pages = UsageHelper.get_aggregated_pages_processed(
+ run_id=file_execution_id
+ )
+
+ return JsonResponse(
+ {
+ "success": True,
+ "data": {
+ "file_execution_id": file_execution_id,
+ "total_pages_processed": total_pages,
+ },
+ }
+ )
+
+ except Exception as e:
+ logger.error(
+ f"Error getting pages processed for file_execution_id {file_execution_id}: {e}",
+ exc_info=True,
+ )
+ return JsonResponse(
+ {
+ "success": False,
+ "error": "Internal server error",
+ "file_execution_id": file_execution_id,
+ },
+ status=status.HTTP_500_INTERNAL_SERVER_ERROR,
+ )
diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.py
index a06605754..dc19db881 100644
--- a/workers/shared/api/internal_client.py
+++ b/workers/shared/api/internal_client.py
@@ -1536,6 +1536,22 @@ class InternalAPIClient(CachedAPIClientMixin):
)
return None
+ def get_aggregated_pages_processed(
+ self, file_execution_id: str, organization_id: str | None = None
+ ) -> int | None:
+ """Get aggregated pages processed for a file execution.
+
+ Args:
+ file_execution_id: File execution ID to get pages data for
+ organization_id: Optional organization ID override
+
+ Returns:
+ Total pages processed count, or None if no data found
+ """
+ return self.usage_client.get_aggregated_pages_processed(
+ file_execution_id, organization_id
+ )
+
# Export all classes and exceptions for backward compatibility
# =============================
# CACHE MANAGEMENT UTILITIES
diff --git a/workers/shared/clients/usage_client.py b/workers/shared/clients/usage_client.py
index 39c30fcc9..a7ba8c701 100644
--- a/workers/shared/clients/usage_client.py
+++ b/workers/shared/clients/usage_client.py
@@ -170,3 +170,30 @@ class UsageAPIClient(BaseAPIClient, UsageOperationMixin):
file_execution_id=validated_file_execution_id,
message="Failed to retrieve usage data",
)
+
+ def get_aggregated_pages_processed(
+ self, file_execution_id: str | uuid.UUID, organization_id: str | None = None
+ ) -> int | None:
+ """Get aggregated pages processed for a file execution.
+
+ Args:
+ file_execution_id: File execution ID to get pages data for
+ organization_id: Optional organization ID override
+
+ Returns:
+ Total pages processed count, or None if no data found
+ """
+ try:
+ validated_id = self._validate_file_execution_id(file_execution_id)
+ endpoint = f"v1/usage/aggregated-pages-processed/{validated_id}/"
+ response = self.get(endpoint, organization_id=organization_id)
+
+ if response and response.get("success"):
+ return response.get("data", {}).get("total_pages_processed")
+ return None
+
+ except Exception as e:
+ logger.error(
+ f"Failed to get pages processed for {file_execution_id}: {str(e)}"
+ )
+ return None
diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py
index 9207556db..7747ae5db 100644
--- a/workers/shared/workflow/destination_connector.py
+++ b/workers/shared/workflow/destination_connector.py
@@ -629,8 +629,9 @@ class WorkerDestinationConnector:
f"🔌 File '{file_ctx.file_name}' marked for API processing - preparing response",
)
- # Prepare metadata - add HITL info only if plugin is available (cloud feature)
- api_metadata = result.metadata.copy() if result.metadata else {}
+ # Enrich metadata with usage and pages processed data
+ api_metadata = self.get_combined_metadata(exec_ctx.api_client, result.metadata)
+ # Add HITL info only if plugin is available (cloud feature)
if has_manual_review_plugin():
api_metadata["hitl"] = {
"file_sent_to_hitl": result.has_hitl,
@@ -900,11 +901,19 @@ class WorkerDestinationConnector:
Returns:
dict[str, Any]: Combined metadata including workflow and usage data.
"""
+ if metadata is None:
+ metadata = {}
+ if not api_client:
+ return metadata
file_execution_id = self.file_execution_id
usage_metadata = api_client.get_aggregated_token_count(file_execution_id)
- if metadata and usage_metadata:
+ if usage_metadata:
metadata["usage"] = usage_metadata.to_dict()
+ if file_execution_id:
+ metadata["total_pages_processed"] = api_client.get_aggregated_pages_processed(
+ file_execution_id
+ )
return metadata
def insert_into_db(