Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

[FIX] Optimize queries made by worker and retry config of worker base client (#1798)

Chandrasekharan Md7babecf0355
diff --git a/backend/workflow_manager/file_execution/internal_views.py b/backend/workflow_manager/file_execution/internal_views.py
index fbd5c65e7..98bf086c3 100644
--- a/backend/workflow_manager/file_execution/internal_views.py
+++ b/backend/workflow_manager/file_execution/internal_views.py
@@ -7,6 +7,7 @@ import logging
from django.db import transaction
from rest_framework import status, viewsets
from rest_framework.decorators import action
+from rest_framework.exceptions import APIException, NotFound
from rest_framework.response import Response
from rest_framework.views import APIView
from utils.organization_utils import filter_queryset_by_organization
@@ -29,134 +30,62 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):
serializer_class = WorkflowFileExecutionSerializer
lookup_field = "id"
- def get_queryset(self):
- """Get file executions filtered by organization context and query parameters."""
- queryset = WorkflowFileExecution.objects.all()
+ def get_object(self):
+ """Retrieve a single file execution by PK, bypassing get_queryset().
- # Filter through the relationship: WorkflowFileExecution -> WorkflowExecution -> Workflow -> Organization
+ This avoids the expensive filtered queryset (3-table JOIN + query params)
+ for single-object operations (retrieve, update, destroy).
+ """
+ pk = self.kwargs.get(self.lookup_field)
+ queryset = WorkflowFileExecution.objects.filter(id=pk)
queryset = filter_queryset_by_organization(
queryset, self.request, "workflow_execution__workflow__organization"
)
+ try:
+ obj = queryset.get()
+ except WorkflowFileExecution.DoesNotExist as exc:
+ raise NotFound(f"No file execution found with ID {pk}") from exc
+ self.check_object_permissions(self.request, obj)
+ return obj
- # Debug: Log initial queryset count after organization filtering
- org_filtered_count = queryset.count()
- logger.debug(
- f"After organization filtering: {org_filtered_count} file executions"
+ def get_queryset(self):
+ """Get file executions filtered by organization context and query parameters.
+
+ Only used for list() — single-object lookups go through get_object().
+ """
+ queryset = WorkflowFileExecution.objects.all()
+ queryset = filter_queryset_by_organization(
+ queryset, self.request, "workflow_execution__workflow__organization"
)
- # Support filtering by query parameters for get-or-create operations
execution_id = self.request.query_params.get("execution_id")
file_hash = self.request.query_params.get("file_hash")
provider_file_uuid = self.request.query_params.get("provider_file_uuid")
workflow_id = self.request.query_params.get("workflow_id")
- file_path = self.request.query_params.get(
- "file_path"
- ) # CRITICAL: Add file_path parameter
-
- logger.debug(
- f"Query parameters: execution_id={execution_id}, file_hash={file_hash}, provider_file_uuid={provider_file_uuid}, workflow_id={workflow_id}, file_path={file_path}"
- )
+ file_path = self.request.query_params.get("file_path")
- # Apply filters step by step with debugging
if execution_id:
queryset = queryset.filter(workflow_execution_id=execution_id)
- logger.info(
- f"DEBUG: After execution_id filter: {queryset.count()} file executions"
- )
- # CRITICAL FIX: Include file_path filter to match unique constraints
if file_path:
queryset = queryset.filter(file_path=file_path)
- logger.debug(f"After file_path filter: {queryset.count()} file executions")
- # CRITICAL FIX: Match backend manager logic - use file_hash OR provider_file_uuid (not both)
+ # Use file_hash OR provider_file_uuid (not both) to match manager logic
if file_hash:
queryset = queryset.filter(file_hash=file_hash)
- logger.info(
- f"DEBUG: After file_hash filter: {queryset.count()} file executions"
- )
elif provider_file_uuid:
queryset = queryset.filter(provider_file_uuid=provider_file_uuid)
- logger.info(
- f"DEBUG: After provider_file_uuid filter: {queryset.count()} file executions"
- )
if workflow_id:
queryset = queryset.filter(workflow_execution__workflow_id=workflow_id)
- logger.info(
- f"DEBUG: After workflow_id filter: {queryset.count()} file executions"
- )
-
- final_count = queryset.count()
- logger.info(
- f"Final queryset count: {final_count} file executions for params: execution_id={execution_id}, file_hash={file_hash}, provider_file_uuid={provider_file_uuid}, workflow_id={workflow_id}, file_path={file_path}"
- )
-
- # If we still have too many results, something is wrong with the filtering
- if final_count > 10: # Reasonable threshold
- logger.warning(
- f"Query returned {final_count} file executions - filtering may not be working correctly"
- )
- logger.warning(
- f"Query params: execution_id={execution_id}, file_hash={file_hash}, workflow_id={workflow_id}"
- )
return queryset
- def list(self, request, *args, **kwargs):
- """List file executions with enhanced filtering validation."""
- queryset = self.get_queryset()
- count = queryset.count()
-
- # If we get too many results, it means the filtering failed
- if count > 50: # Conservative threshold
- logger.error(
- f"GET request returned {count} file executions - this suggests broken query parameter filtering"
- )
- logger.error(f"Request query params: {dict(request.query_params)}")
-
- # For debugging, show a sample of what we're returning
- sample_ids = list(queryset.values_list("id", flat=True)[:5])
- logger.error(f"Sample file execution IDs: {sample_ids}")
-
- # Return error response instead of broken list
- return Response(
- {
- "error": "Query returned too many results",
- "detail": f"Expected 0-1 file executions but got {count}. Check query parameters.",
- "count": count,
- "query_params": dict(request.query_params),
- },
- status=status.HTTP_400_BAD_REQUEST,
- )
-
- # Continue with normal list behavior for reasonable result counts
- logger.info(f"GET request successfully filtered to {count} file executions")
- return super().list(request, *args, **kwargs)
-
@action(detail=True, methods=["post"])
def status(self, request, id=None):
"""Update file execution status."""
try:
- # Get file execution by ID with organization filtering
- # Don't use self.get_object() as it applies query parameter filtering
- base_queryset = WorkflowFileExecution.objects.all()
- base_queryset = filter_queryset_by_organization(
- base_queryset, request, "workflow_execution__workflow__organization"
- )
-
- try:
- file_execution = base_queryset.get(id=id)
- except WorkflowFileExecution.DoesNotExist:
- logger.warning(f"WorkflowFileExecution {id} not found for status update")
- return Response(
- {
- "error": "WorkflowFileExecution not found",
- "detail": f"No file execution record found with ID {id}",
- },
- status=status.HTTP_404_NOT_FOUND,
- )
-
+ file_execution = self.get_object()
serializer = FileExecutionStatusUpdateSerializer(data=request.data)
if serializer.is_valid():
@@ -193,6 +122,8 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):
return Response(serializer.errors, status=status.HTTP_400_BAD_REQUEST)
+ except APIException:
+ raise
except Exception as e:
logger.error(f"Failed to update file execution status {id}: {str(e)}")
return Response(
@@ -287,23 +218,7 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):
def update_hash(self, request, id=None):
"""Update file execution with computed file hash."""
try:
- # Get file execution by ID with organization filtering
- base_queryset = WorkflowFileExecution.objects.all()
- base_queryset = filter_queryset_by_organization(
- base_queryset, request, "workflow_execution__workflow__organization"
- )
-
- try:
- file_execution = base_queryset.get(id=id)
- except WorkflowFileExecution.DoesNotExist:
- logger.warning(f"WorkflowFileExecution {id} not found for hash update")
- return Response(
- {
- "error": "WorkflowFileExecution not found",
- "detail": f"No file execution record found with ID {id}",
- },
- status=status.HTTP_404_NOT_FOUND,
- )
+ file_execution = self.get_object()
# Extract update data
file_hash = request.data.get("file_hash")
@@ -336,6 +251,8 @@ class FileExecutionInternalViewSet(viewsets.ModelViewSet):
status=status.HTTP_200_OK,
)
+ except APIException:
+ raise
except Exception as e:
logger.error(f"Failed to update file execution hash {id}: {str(e)}")
return Response(
diff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.py
index 971ffb845..a06605754 100644
--- a/workers/shared/api/internal_client.py
+++ b/workers/shared/api/internal_client.py
@@ -431,6 +431,10 @@ class InternalAPIClient(CachedAPIClientMixin):
response_dict = response
return WorkflowExecutionResponse.from_api_response(response_dict)
+ @with_cache(
+ CacheType.WORKFLOW,
+ lambda self, workflow_id, organization_id=None: str(workflow_id),
+ )
def get_workflow(
self, workflow_id: str | uuid.UUID, organization_id: str | None = None
) -> WorkflowDefinitionResponseData:
@@ -462,7 +466,7 @@ class InternalAPIClient(CachedAPIClientMixin):
return response.data
@with_cache(
- CacheType.WORKFLOW,
+ CacheType.WORKFLOW_DEFINITION,
lambda self, workflow_id, organization_id=None: str(workflow_id),
)
def get_workflow_definition(
@@ -483,28 +487,29 @@ class InternalAPIClient(CachedAPIClientMixin):
response_dict = response
return WorkflowDefinitionResponse.from_api_response(response_dict)
+ @with_cache(
+ CacheType.PIPELINE,
+ lambda self, pipeline_id, organization_id=None: str(pipeline_id),
+ )
def get_pipeline_type(
self, pipeline_id: str | uuid.UUID, organization_id: str | None = None
) -> APIResponse:
"""Get pipeline type by checking APIDeployment and Pipeline models."""
return self.execution_client.get_pipeline_type(pipeline_id, organization_id)
+ @with_cache(
+ CacheType.PIPELINE_DATA,
+ lambda self, pipeline_id, check_active=True, organization_id=None: (
+ f"{pipeline_id}:{check_active}"
+ ),
+ )
def get_pipeline_data(
self,
pipeline_id: str | uuid.UUID,
check_active: bool = True,
organization_id: str | None = None,
) -> APIResponse:
- """Get pipeline data by checking APIDeployment and Pipeline models.
-
- Args:
- pipeline_id: Pipeline ID
- check_active: Whether to check if pipeline is active (default: True)
- organization_id: Optional organization ID override
-
- Returns:
- APIResponse containing pipeline data
- """
+ """Get pipeline data by checking APIDeployment and Pipeline models."""
return self.execution_client.get_pipeline_data(
pipeline_id=pipeline_id,
check_active=check_active,
diff --git a/workers/shared/cache/cache_decorator.py b/workers/shared/cache/cache_decorator.py
index 044b9240f..73a5c4e73 100644
--- a/workers/shared/cache/cache_decorator.py
+++ b/workers/shared/cache/cache_decorator.py
@@ -81,6 +81,14 @@ def with_cache(
cache_key = CacheKeyGenerator.workflow_endpoints_key(
cache_key_suffix
)
+ elif cache_type_str == CacheType.WORKFLOW_DEFINITION.value:
+ cache_key = CacheKeyGenerator.workflow_definition_key(
+ cache_key_suffix
+ )
+ elif cache_type_str == CacheType.PIPELINE.value:
+ cache_key = CacheKeyGenerator.pipeline_key(cache_key_suffix)
+ elif cache_type_str == CacheType.PIPELINE_DATA.value:
+ cache_key = CacheKeyGenerator.pipeline_data_key(cache_key_suffix)
else:
cache_key = CacheKeyGenerator.custom_key(
cache_type_str, cache_key_suffix
diff --git a/workers/shared/cache/cache_keys.py b/workers/shared/cache/cache_keys.py
index 1ae71bb9a..0383282e9 100644
--- a/workers/shared/cache/cache_keys.py
+++ b/workers/shared/cache/cache_keys.py
@@ -15,11 +15,21 @@ class CacheKeyGenerator:
"""Generate cache key for workflow data."""
return f"worker_cache:workflow:{workflow_id}"
+ @staticmethod
+ def workflow_definition_key(workflow_id: str) -> str:
+ """Generate cache key for workflow definition (full response)."""
+ return f"worker_cache:workflow_definition:{workflow_id}"
+
@staticmethod
def pipeline_key(pipeline_id: str) -> str:
- """Generate cache key for pipeline data."""
+ """Generate cache key for pipeline type."""
return f"worker_cache:pipeline:{pipeline_id}"
+ @staticmethod
+ def pipeline_data_key(pipeline_id: str) -> str:
+ """Generate cache key for pipeline data."""
+ return f"worker_cache:pipeline_data:{pipeline_id}"
+
@staticmethod
def api_deployment_key(api_id: str, org_id: str) -> str:
"""Generate cache key for API deployment data."""
diff --git a/workers/shared/clients/base_client.py b/workers/shared/clients/base_client.py
index 45d37d816..69759cd3a 100644
--- a/workers/shared/clients/base_client.py
+++ b/workers/shared/clients/base_client.py
@@ -136,15 +136,18 @@ class BaseAPIClient:
def _setup_session(self):
"""Configure session with retry strategy, timeouts, and connection pooling."""
- # Enhanced retry strategy with enum-based status codes
- retry_status_codes = [429, 500, 502, 503, 504]
allowed_http_methods = [method.value for method in HTTPMethod]
+ # Only retry transport-level connection failures at this layer.
+ # Status code retries (5xx, 429) are handled exclusively by _make_request()
+ # to avoid retry amplification (urllib3 retries × app-level retries).
retry_strategy = Retry(
total=self.config.api_retry_attempts,
backoff_factor=self.config.api_retry_backoff_factor,
- status_forcelist=retry_status_codes,
+ status_forcelist=[],
allowed_methods=allowed_http_methods,
+ connect=self.config.api_retry_attempts,
+ read=0,
respect_retry_after_header=True,
)
@@ -200,8 +203,8 @@ class BaseAPIClient:
data: dict[str, Any] | None = None,
params: dict[str, Any] | None = None,
timeout: int | None = None,
- max_retries: int = 3,
- backoff_factor: float = 0.5,
+ max_retries: int | None = None,
+ backoff_factor: float | None = None,
organization_id: str | None = None,
) -> dict[str, Any]:
"""Enhanced HTTP request with robust error handling and retry logic.
@@ -212,8 +215,8 @@ class BaseAPIClient:
data: Request payload for POST/PUT/PATCH
params: Query parameters
timeout: Request timeout in seconds
- max_retries: Maximum number of retry attempts
- backoff_factor: Exponential backoff factor
+ max_retries: Maximum number of retry attempts (defaults to config)
+ backoff_factor: Exponential backoff factor (defaults to config)
organization_id: Optional organization ID override
Returns:
@@ -225,6 +228,14 @@ class BaseAPIClient:
"""
url = f"{self.base_url.rstrip('/')}/{endpoint.lstrip('/')}"
timeout = timeout or self.config.api_timeout
+ max_retries = (
+ max_retries if max_retries is not None else self.config.api_retry_attempts
+ )
+ backoff_factor = (
+ backoff_factor
+ if backoff_factor is not None
+ else self.config.api_retry_backoff_factor
+ )
last_exception = None
# HTTP status codes that should trigger retries