LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[FIX] Optimize queries made by worker and retry config of worker base client (#1798)
Chandrasekharan Md7babecf0355
diff --git a/backend/workflow_manager/file_execution/internal_views.py b/backend/workflow_manager/file_execution/internal_views.pyindex fbd5c65e7..98bf086c3 100644--- a/backend/workflow_manager/file_execution/internal_views.py+++ b/backend/workflow_manager/file_execution/internal_views.py@@ -7,6 +7,7 @@ import loggingfrom django.db import transactionfrom rest_framework import status, viewsetsfrom rest_framework.decorators import action+from rest_framework.exceptions import APIException, NotFoundfrom rest_framework.response import Responsefrom rest_framework.views import APIViewfrom utils.organization_utils import filter_queryset_by_organization@@ -29,134 +30,62 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):serializer_class = WorkflowFileExecutionSerializerlookup_field = "id"- def get_queryset(self):- """Get file executions filtered by organization context and query parameters."""- queryset = WorkflowFileExecution.objects.all()+ def get_object(self):+ """Retrieve a single file execution by PK, bypassing get_queryset().- # Filter through the relationship: WorkflowFileExecution -> WorkflowExecution -> Workflow -> Organization+ This avoids the expensive filtered queryset (3-table JOIN + query params)+ for single-object operations (retrieve, update, destroy).+ """+ pk = self.kwargs.get(self.lookup_field)+ queryset = WorkflowFileExecution.objects.filter(id=pk)queryset = filter_queryset_by_organization(queryset, self.request, "workflow_execution__workflow__organization")+ try:+ obj = queryset.get()+ except WorkflowFileExecution.DoesNotExist as exc:+ raise NotFound(f"No file execution found with ID {pk}") from exc+ self.check_object_permissions(self.request, obj)+ return obj- # Debug: Log initial queryset count after organization filtering- org_filtered_count = queryset.count()- logger.debug(- f"After organization filtering: {org_filtered_count} file executions"+ def get_queryset(self):+ """Get file executions filtered by organization context and query parameters.++ Only used for list() — single-object lookups go through get_object().+ """+ queryset = WorkflowFileExecution.objects.all()+ queryset = filter_queryset_by_organization(+ queryset, self.request, "workflow_execution__workflow__organization")- # Support filtering by query parameters for get-or-create operationsexecution_id = self.request.query_params.get("execution_id")file_hash = self.request.query_params.get("file_hash")provider_file_uuid = self.request.query_params.get("provider_file_uuid")workflow_id = self.request.query_params.get("workflow_id")- file_path = self.request.query_params.get(- "file_path"- ) # CRITICAL: Add file_path parameter-- logger.debug(- f"Query parameters: execution_id={execution_id}, file_hash={file_hash}, provider_file_uuid={provider_file_uuid}, workflow_id={workflow_id}, file_path={file_path}"- )+ file_path = self.request.query_params.get("file_path")- # Apply filters step by step with debuggingif execution_id:queryset = queryset.filter(workflow_execution_id=execution_id)- logger.info(- f"DEBUG: After execution_id filter: {queryset.count()} file executions"- )- # CRITICAL FIX: Include file_path filter to match unique constraintsif file_path:queryset = queryset.filter(file_path=file_path)- logger.debug(f"After file_path filter: {queryset.count()} file executions")- # CRITICAL FIX: Match backend manager logic - use file_hash OR provider_file_uuid (not both)+ # Use file_hash OR provider_file_uuid (not both) to match manager logicif file_hash:queryset = queryset.filter(file_hash=file_hash)- logger.info(- f"DEBUG: After file_hash filter: {queryset.count()} file executions"- )elif provider_file_uuid:queryset = queryset.filter(provider_file_uuid=provider_file_uuid)- logger.info(- f"DEBUG: After provider_file_uuid filter: {queryset.count()} file executions"- )if workflow_id:queryset = queryset.filter(workflow_execution__workflow_id=workflow_id)- logger.info(- f"DEBUG: After workflow_id filter: {queryset.count()} file executions"- )-- final_count = queryset.count()- logger.info(- f"Final queryset count: {final_count} file executions for params: execution_id={execution_id}, file_hash={file_hash}, provider_file_uuid={provider_file_uuid}, workflow_id={workflow_id}, file_path={file_path}"- )-- # If we still have too many results, something is wrong with the filtering- if final_count > 10: # Reasonable threshold- logger.warning(- f"Query returned {final_count} file executions - filtering may not be working correctly"- )- logger.warning(- f"Query params: execution_id={execution_id}, file_hash={file_hash}, workflow_id={workflow_id}"- )return queryset- def list(self, request, *args, **kwargs):- """List file executions with enhanced filtering validation."""- queryset = self.get_queryset()- count = queryset.count()-- # If we get too many results, it means the filtering failed- if count > 50: # Conservative threshold- logger.error(- f"GET request returned {count} file executions - this suggests broken query parameter filtering"- )- logger.error(f"Request query params: {dict(request.query_params)}")-- # For debugging, show a sample of what we're returning- sample_ids = list(queryset.values_list("id", flat=True)[:5])- logger.error(f"Sample file execution IDs: {sample_ids}")-- # Return error response instead of broken list- return Response(- {- "error": "Query returned too many results",- "detail": f"Expected 0-1 file executions but got {count}. Check query parameters.",- "count": count,- "query_params": dict(request.query_params),- },- status=status.HTTP_400_BAD_REQUEST,- )-- # Continue with normal list behavior for reasonable result counts- logger.info(f"GET request successfully filtered to {count} file executions")- return super().list(request, *args, **kwargs)-@action(detail=True, methods=["post"])def status(self, request, id=None):"""Update file execution status."""try:- # Get file execution by ID with organization filtering- # Don't use self.get_object() as it applies query parameter filtering- base_queryset = WorkflowFileExecution.objects.all()- base_queryset = filter_queryset_by_organization(- base_queryset, request, "workflow_execution__workflow__organization"- )-- try:- file_execution = base_queryset.get(id=id)- except WorkflowFileExecution.DoesNotExist:- logger.warning(f"WorkflowFileExecution {id} not found for status update")- return Response(- {- "error": "WorkflowFileExecution not found",- "detail": f"No file execution record found with ID {id}",- },- status=status.HTTP_404_NOT_FOUND,- )-+ file_execution = self.get_object()serializer = FileExecutionStatusUpdateSerializer(data=request.data)if serializer.is_valid():@@ -193,6 +122,8 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)+ except APIException:+ raiseexcept Exception as e:logger.error(f"Failed to update file execution status {id}: {str(e)}")return Response(@@ -287,23 +218,7 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):def update_hash(self, request, id=None):"""Update file execution with computed file hash."""try:- # Get file execution by ID with organization filtering- base_queryset = WorkflowFileExecution.objects.all()- base_queryset = filter_queryset_by_organization(- base_queryset, request, "workflow_execution__workflow__organization"- )-- try:- file_execution = base_queryset.get(id=id)- except WorkflowFileExecution.DoesNotExist:- logger.warning(f"WorkflowFileExecution {id} not found for hash update")- return Response(- {- "error": "WorkflowFileExecution not found",- "detail": f"No file execution record found with ID {id}",- },- status=status.HTTP_404_NOT_FOUND,- )+ file_execution = self.get_object()# Extract update datafile_hash = request.data.get("file_hash")@@ -336,6 +251,8 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):status=status.HTTP_200_OK,)+ except APIException:+ raiseexcept Exception as e:logger.error(f"Failed to update file execution hash {id}: {str(e)}")return Response(diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.pyindex 971ffb845..a06605754 100644--- a/workers/shared/api/internal_client.py+++ b/workers/shared/api/internal_client.py@@ -431,6 +431,10 @@ class InternalAPIClient(CachedAPIClientMixin):response_dict = responsereturn WorkflowExecutionResponse.from_api_response(response_dict)+ @with_cache(+ CacheType.WORKFLOW,+ lambda self, workflow_id, organization_id=None: str(workflow_id),+ )def get_workflow(self, workflow_id: str | uuid.UUID, organization_id: str | None = None) -> WorkflowDefinitionResponseData:@@ -462,7 +466,7 @@ class InternalAPIClient(CachedAPIClientMixin):return response.data@with_cache(- CacheType.WORKFLOW,+ CacheType.WORKFLOW_DEFINITION,lambda self, workflow_id, organization_id=None: str(workflow_id),)def get_workflow_definition(@@ -483,28 +487,29 @@ class InternalAPIClient(CachedAPIClientMixin):response_dict = responsereturn WorkflowDefinitionResponse.from_api_response(response_dict)+ @with_cache(+ CacheType.PIPELINE,+ lambda self, pipeline_id, organization_id=None: str(pipeline_id),+ )def get_pipeline_type(self, pipeline_id: str | uuid.UUID, organization_id: str | None = None) -> APIResponse:"""Get pipeline type by checking APIDeployment and Pipeline models."""return self.execution_client.get_pipeline_type(pipeline_id, organization_id)+ @with_cache(+ CacheType.PIPELINE_DATA,+ lambda self, pipeline_id, check_active=True, organization_id=None: (+ f"{pipeline_id}:{check_active}"+ ),+ )def get_pipeline_data(self,pipeline_id: str | uuid.UUID,check_active: bool = True,organization_id: str | None = None,) -> APIResponse:- """Get pipeline data by checking APIDeployment and Pipeline models.-- Args:- pipeline_id: Pipeline ID- check_active: Whether to check if pipeline is active (default: True)- organization_id: Optional organization ID override-- Returns:- APIResponse containing pipeline data- """+ """Get pipeline data by checking APIDeployment and Pipeline models."""return self.execution_client.get_pipeline_data(pipeline_id=pipeline_id,check_active=check_active,diff --git a/workers/shared/cache/cache_decorator.py b/workers/shared/cache/cache_decorator.pyindex 044b9240f..73a5c4e73 100644--- a/workers/shared/cache/cache_decorator.py+++ b/workers/shared/cache/cache_decorator.py@@ -81,6 +81,14 @@ def with_cache(cache_key = CacheKeyGenerator.workflow_endpoints_key(cache_key_suffix)+ elif cache_type_str == CacheType.WORKFLOW_DEFINITION.value:+ cache_key = CacheKeyGenerator.workflow_definition_key(+ cache_key_suffix+ )+ elif cache_type_str == CacheType.PIPELINE.value:+ cache_key = CacheKeyGenerator.pipeline_key(cache_key_suffix)+ elif cache_type_str == CacheType.PIPELINE_DATA.value:+ cache_key = CacheKeyGenerator.pipeline_data_key(cache_key_suffix)else:cache_key = CacheKeyGenerator.custom_key(cache_type_str, cache_key_suffixdiff --git a/workers/shared/cache/cache_keys.py b/workers/shared/cache/cache_keys.pyindex 1ae71bb9a..0383282e9 100644--- a/workers/shared/cache/cache_keys.py+++ b/workers/shared/cache/cache_keys.py@@ -15,11 +15,21 @@ class CacheKeyGenerator:"""Generate cache key for workflow data."""return f"worker_cache:workflow:{workflow_id}"+ @staticmethod+ def workflow_definition_key(workflow_id: str) -> str:+ """Generate cache key for workflow definition (full response)."""+ return f"worker_cache:workflow_definition:{workflow_id}"+@staticmethoddef pipeline_key(pipeline_id: str) -> str:- """Generate cache key for pipeline data."""+ """Generate cache key for pipeline type."""return f"worker_cache:pipeline:{pipeline_id}"+ @staticmethod+ def pipeline_data_key(pipeline_id: str) -> str:+ """Generate cache key for pipeline data."""+ return f"worker_cache:pipeline_data:{pipeline_id}"+@staticmethoddef api_deployment_key(api_id: str, org_id: str) -> str:"""Generate cache key for API deployment data."""diff --git a/workers/shared/clients/base_client.py b/workers/shared/clients/base_client.pyindex 45d37d816..69759cd3a 100644--- a/workers/shared/clients/base_client.py+++ b/workers/shared/clients/base_client.py@@ -136,15 +136,18 @@ class BaseAPIClient:def _setup_session(self):"""Configure session with retry strategy, timeouts, and connection pooling."""- # Enhanced retry strategy with enum-based status codes- retry_status_codes = [429, 500, 502, 503, 504]allowed_http_methods = [method.value for method in HTTPMethod]+ # Only retry transport-level connection failures at this layer.+ # Status code retries (5xx, 429) are handled exclusively by _make_request()+ # to avoid retry amplification (urllib3 retries × app-level retries).retry_strategy = Retry(total=self.config.api_retry_attempts,backoff_factor=self.config.api_retry_backoff_factor,- status_forcelist=retry_status_codes,+ status_forcelist=[],allowed_methods=allowed_http_methods,+ connect=self.config.api_retry_attempts,+ read=0,respect_retry_after_header=True,)@@ -200,8 +203,8 @@ class BaseAPIClient:data: dict[str, Any] | None = None,params: dict[str, Any] | None = None,timeout: int | None = None,- max_retries: int = 3,- backoff_factor: float = 0.5,+ max_retries: int | None = None,+ backoff_factor: float | None = None,organization_id: str | None = None,) -> dict[str, Any]:"""Enhanced HTTP request with robust error handling and retry logic.@@ -212,8 +215,8 @@ class BaseAPIClient:data: Request payload for POST/PUT/PATCHparams: Query parameterstimeout: Request timeout in seconds- max_retries: Maximum number of retry attempts- backoff_factor: Exponential backoff factor+ max_retries: Maximum number of retry attempts (defaults to config)+ backoff_factor: Exponential backoff factor (defaults to config)organization_id: Optional organization ID overrideReturns:@@ -225,6 +228,14 @@ class BaseAPIClient:"""url = f"{self.base_url.rstrip('/')}/{endpoint.lstrip('/')}"timeout = timeout or self.config.api_timeout+ max_retries = (+ max_retries if max_retries is not None else self.config.api_retry_attempts+ )+ backoff_factor = (+ backoff_factor+ if backoff_factor is not None+ else self.config.api_retry_backoff_factor+ )last_exception = None# HTTP status codes that should trigger retries