LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-3211 [FEAT] HTTP session lifecycle management for workers API clients (#1782)
alidf3a5605620f
diff --git a/workers/api-deployment/tasks.py b/workers/api-deployment/tasks.pyindex 5345d41d5..6406a864e 100644--- a/workers/api-deployment/tasks.py+++ b/workers/api-deployment/tasks.py@@ -168,13 +168,30 @@ def _unified_api_execution(Returns:Execution result dictionary"""+ # Set up execution context - exit early if setup fails+ organization_id = schema_nametry:- # Set up execution context using shared utilities- organization_id = schema_name- config, api_client = WorkerExecutionContext.setup_execution_context(+ _, api_client = WorkerExecutionContext.setup_execution_context(organization_id, execution_id, workflow_id)+ except Exception as e:+ logger.error(f"Failed to setup execution context: {e}")+ # Clean up StateStore since setup_execution_context may have+ # partially populated it before failing+ try:+ from shared.infrastructure.context import StateStore+ StateStore.clear_all()+ except Exception:+ logger.debug("StateStore cleanup also failed during early exit")+ return {+ "execution_id": execution_id,+ "status": "ERROR",+ "error": str(e),+ "files_processed": 0,+ }++ try:# Log task start with standardized formatWorkerExecutionContext.log_task_start(f"unified_api_execution_{task_type}",@@ -216,12 +233,12 @@ def _unified_api_execution(schema_name=organization_id,workflow_id=workflow_id,execution_id=execution_id,- hash_values_of_files=converted_files, # Changed parameter name+ hash_values_of_files=converted_files,scheduled=scheduled,execution_mode=execution_mode,pipeline_id=pipeline_id,use_file_history=use_file_history,- task_id=task_instance.request.id, # Add required task_id+ task_id=task_instance.request.id,**kwargs,)@@ -233,25 +250,14 @@ def _unified_api_execution(f"files_processed={len(converted_files)}",)- # CRITICAL: Clean up StateStore to prevent data leaks between tasks- try:- from shared.infrastructure.context import StateStore-- StateStore.clear_all()- logger.debug("🧹 Cleaned up StateStore context to prevent data leaks")- except Exception as cleanup_error:- logger.warning(f"Failed to cleanup StateStore context: {cleanup_error}")-return resultexcept Exception as e:logger.error(f"API execution failed: {e}")- # Handle execution error with standardized pattern- if "api_client" in locals():- WorkerExecutionContext.handle_execution_error(- api_client, execution_id, e, logger, f"api_execution_{task_type}"- )+ WorkerExecutionContext.handle_execution_error(+ api_client, execution_id, e, logger, f"api_execution_{task_type}"+ )# Log completion with errorWorkerExecutionContext.log_task_completion(@@ -261,19 +267,6 @@ def _unified_api_execution(f"error={str(e)}",)- # CRITICAL: Clean up StateStore to prevent data leaks between tasks (error path)- try:- from shared.infrastructure.context import StateStore-- StateStore.clear_all()- logger.debug(- "🧹 Cleaned up StateStore context to prevent data leaks (error path)"- )- except Exception as cleanup_error:- logger.warning(- f"Failed to cleanup StateStore context on error: {cleanup_error}"- )-return {"execution_id": execution_id,"status": "ERROR",@@ -281,6 +274,20 @@ def _unified_api_execution("files_processed": 0,}+ finally:+ try:+ api_client.close()+ except Exception as e:+ logger.warning("api_client.close() failed during cleanup: %s", e)++ # Clean up StateStore to prevent data leaks between tasks+ try:+ from shared.infrastructure.context import StateStore++ StateStore.clear_all()+ except Exception as cleanup_error:+ logger.warning(f"Failed to cleanup StateStore context: {cleanup_error}")+@app.task(bind=True,diff --git a/workers/callback/tasks.py b/workers/callback/tasks.pyindex 1e83559bb..d7252b9d2 100644--- a/workers/callback/tasks.py+++ b/workers/callback/tasks.py@@ -1386,123 +1386,130 @@ def _process_batch_callback_core(f"organization_id={context.organization_id}, api_client={context.api_client is not None}")raise RuntimeError(f"Invalid context for execution {context.execution_id}")- with log_context(- task_id=context.task_id,- execution_id=context.execution_id,- workflow_id=context.workflow_id,- organization_id=context.organization_id,- pipeline_id=context.pipeline_id,- ):- logger.info(- f"Starting batch callback processing for execution {context.execution_id}"- )+ try:+ with log_context(+ task_id=context.task_id,+ execution_id=context.execution_id,+ workflow_id=context.workflow_id,+ organization_id=context.organization_id,+ pipeline_id=context.pipeline_id,+ ):+ logger.info(+ f"Starting batch callback processing for execution {context.execution_id}"+ )- try:- # Use unified status determination with timeout detection (shared with API callback)- aggregated_results, execution_status, expected_files = (- _determine_execution_status_unified(- file_batch_results=results,+ try:+ # Use unified status determination with timeout detection (shared with API callback)+ aggregated_results, execution_status, expected_files = (+ _determine_execution_status_unified(+ file_batch_results=results,+ api_client=context.api_client,+ execution_id=context.execution_id,+ organization_id=context.organization_id,+ )+ )+ # Update workflow execution status using unified function+ execution_update_result = _update_execution_status_unified(api_client=context.api_client,execution_id=context.execution_id,+ final_status=execution_status,+ aggregated_results=aggregated_results,organization_id=context.organization_id,+ error_message=None,+ )+ # Handle pipeline updates using unified function (non-API deployment)+ pipeline_result = _handle_pipeline_updates_unified(+ context, execution_status, is_api_deployment=False)- )- # Update workflow execution status using unified function- execution_update_result = _update_execution_status_unified(- api_client=context.api_client,- execution_id=context.execution_id,- final_status=execution_status,- aggregated_results=aggregated_results,- organization_id=context.organization_id,- error_message=None,- )- # Handle pipeline updates using unified function (non-API deployment)- pipeline_result = _handle_pipeline_updates_unified(- context, execution_status, is_api_deployment=False- )-- # Track subscription usage if plugin is present- subscription_tracking_result = _track_subscription_usage_if_available(- context=context,- execution_status=execution_status,- )-- # Add missing UI logs for cost and final workflow status (matching backend behavior)- _publish_final_workflow_ui_logs(- context=context,- aggregated_results=aggregated_results,- execution_status=execution_status,- )- # Handle resource cleanup using existing function- cleanup_result = _cleanup_execution_resources(context)- callback_result = {- "status": "completed",- "execution_id": context.execution_id,- "workflow_id": context.workflow_id,- "task_id": context.task_id,- "aggregated_results": aggregated_results,- "execution_status": execution_status,- "expected_files": expected_files,- "execution_update_result": execution_update_result,- "pipeline_result": pipeline_result,- "subscription_tracking_result": subscription_tracking_result,- "cleanup_result": cleanup_result,- "pipeline_id": context.pipeline_id,- "unified_callback": True,- "shared_timeout_detection": True,- }+ # Track subscription usage if plugin is present+ subscription_tracking_result = _track_subscription_usage_if_available(+ context=context,+ execution_status=execution_status,+ )- logger.info(- f"Completed unified callback processing for execution {context.execution_id} "- f"with status {execution_status}"- )- # Handle notifications using unified function (non-critical)- try:- notification_result = _handle_notifications_unified(- api_client=context.api_client,- status=execution_status,- organization_id=context.organization_id,- execution_id=context.execution_id,- pipeline_id=context.pipeline_id,- workflow_id=context.workflow_id,- pipeline_name=context.pipeline_name,- pipeline_type=context.pipeline_type,- error_message=None,+ # Add missing UI logs for cost and final workflow status (matching backend behavior)+ _publish_final_workflow_ui_logs(+ context=context,+ aggregated_results=aggregated_results,+ execution_status=execution_status,)- callback_result["notification_result"] = notification_result- except Exception as notif_error:- logger.warning(f"Failed to handle notifications: {notif_error}")- callback_result["notification_result"] = {- "status": "failed",- "error": str(notif_error),++ # Handle resource cleanup using existing function+ cleanup_result = _cleanup_execution_resources(context)+ callback_result = {+ "status": "completed",+ "execution_id": context.execution_id,+ "workflow_id": context.workflow_id,+ "task_id": context.task_id,+ "aggregated_results": aggregated_results,+ "execution_status": execution_status,+ "expected_files": expected_files,+ "execution_update_result": execution_update_result,+ "pipeline_result": pipeline_result,+ "subscription_tracking_result": subscription_tracking_result,+ "cleanup_result": cleanup_result,+ "pipeline_id": context.pipeline_id,+ "unified_callback": True,+ "shared_timeout_detection": True,}- return callback_result+ logger.info(+ f"Completed unified callback processing for execution {context.execution_id} "+ f"with status {execution_status}"+ )+ # Handle notifications using unified function (non-critical)+ try:+ notification_result = _handle_notifications_unified(+ api_client=context.api_client,+ status=execution_status,+ organization_id=context.organization_id,+ execution_id=context.execution_id,+ pipeline_id=context.pipeline_id,+ workflow_id=context.workflow_id,+ pipeline_name=context.pipeline_name,+ pipeline_type=context.pipeline_type,+ error_message=None,+ )+ callback_result["notification_result"] = notification_result+ except Exception as notif_error:+ logger.warning(f"Failed to handle notifications: {notif_error}")+ callback_result["notification_result"] = {+ "status": "failed",+ "error": str(notif_error),+ }- except Exception as e:- logger.error(- f"Unified batch callback processing failed for execution {context.execution_id}: {e}"- )+ return callback_result- # Try to mark execution as failed using unified function- try:- _update_execution_status_unified(- context.api_client,- context.execution_id,- ExecutionStatus.ERROR.value,- {"error": str(e)[:500]},- context.organization_id,- error_message=str(e)[:500],- )- logger.info(- f"Marked execution {context.execution_id} as failed using unified function"+ except Exception as e:+ logger.error(+ f"Unified batch callback processing failed for execution {context.execution_id}: {e}")- except Exception as cleanup_error:- logger.error(f"Failed to mark execution as failed: {cleanup_error}")- # Re-raise for Celery retry mechanism- raise+ # Try to mark execution as failed using unified function+ try:+ _update_execution_status_unified(+ context.api_client,+ context.execution_id,+ ExecutionStatus.ERROR.value,+ {"error": str(e)[:500]},+ context.organization_id,+ error_message=str(e)[:500],+ )+ logger.info(+ f"Marked execution {context.execution_id} as failed using unified function"+ )+ except Exception as cleanup_error:+ logger.error(f"Failed to mark execution as failed: {cleanup_error}")++ # Re-raise for Celery retry mechanism+ raise+ finally:+ if context.api_client is not None:+ try:+ context.api_client.close()+ except Exception as e:+ logger.warning("api_client.close() failed during cleanup: %s", e)@app.task(@@ -1581,194 +1588,200 @@ def process_batch_callback_api(api_client = create_api_client(organization_id)logger.info(f"Created organization-scoped API client: {organization_id}")- execution_response = api_client.get_workflow_execution(- execution_id, file_execution=False- )- if not execution_response.success:- raise Exception(f"Failed to get execution context: {execution_response.error}")- execution_context = execution_response.data- workflow_execution = execution_context.get("execution", {})- workflow = execution_context.get("workflow", {})+ try:+ execution_response = api_client.get_workflow_execution(+ execution_id, file_execution=False+ )+ if not execution_response.success:+ raise RuntimeError(+ f"Failed to get execution context: {execution_response.error}"+ )+ execution_context = execution_response.data+ workflow_execution = execution_context.get("execution", {})+ workflow = execution_context.get("workflow", {})- # Extract schema_name and workflow_id from context- schema_name = organization_id # For API callbacks, schema_name = organization_id- workflow_id = workflow_execution.get("workflow_id") or workflow.get("id")+ # Extract schema_name and workflow_id from context+ schema_name = organization_id # For API callbacks, schema_name = organization_id+ workflow_id = workflow_execution.get("workflow_id") or workflow.get("id")- logger.info(- f"Extracted context: schema_name={schema_name}, workflow_id={workflow_id}, pipeline_id={pipeline_id}"- )-- with log_context(- task_id=task_id,- execution_id=execution_id,- workflow_id=workflow_id,- organization_id=organization_id,- pipeline_id=pipeline_id,- ):logger.info(- f"Processing API callback for execution {execution_id} with {len(file_batch_results)} batch results"+ f"Extracted context: schema_name={schema_name}, workflow_id={workflow_id}, pipeline_id={pipeline_id}")- try:- # Create organization-scoped API client using factory pattern- api_client = create_api_client(schema_name)-- # Get pipeline name and type (simplified approach)- if not pipeline_id:- logger.info(- f"No pipeline_id provided for API callback (testing mode). "- f"execution_id={execution_id}, workflow_id={workflow_id}. "- f"APIDeployment-specific updates and notifications will be skipped."- )- pipeline_name = None- pipeline_type = None- else:- # Use simplified pipeline data fetching- pipeline_name, pipeline_type = _fetch_pipeline_data_simplified(- pipeline_id, schema_name, api_client, is_api_deployment=True- )+ with log_context(+ task_id=task_id,+ execution_id=execution_id,+ workflow_id=workflow_id,+ organization_id=organization_id,+ pipeline_id=pipeline_id,+ ):+ logger.info(+ f"Processing API callback for execution {execution_id} with {len(file_batch_results)} batch results"+ )- if pipeline_name:+ try:+ # Get pipeline name and type (simplified approach)+ if not pipeline_id:logger.info(- f"✅ Found pipeline: name='{pipeline_name}', type='{pipeline_type}'"+ f"No pipeline_id provided for API callback (testing mode). "+ f"execution_id={execution_id}, workflow_id={workflow_id}. "+ f"APIDeployment-specific updates and notifications will be skipped.")+ pipeline_name = None+ pipeline_type = Noneelse:- logger.warning(f"Could not fetch pipeline data for {pipeline_id}")- pipeline_name = "Unknown API"- pipeline_type = PipelineType.API.value-- # Use unified status determination with timeout detection- aggregated_results, execution_status, expected_files = (- _determine_execution_status_unified(- file_batch_results=file_batch_results,+ # Use simplified pipeline data fetching+ pipeline_name, pipeline_type = _fetch_pipeline_data_simplified(+ pipeline_id, schema_name, api_client, is_api_deployment=True+ )++ if pipeline_name:+ logger.info(+ f"✅ Found pipeline: name='{pipeline_name}', type='{pipeline_type}'"+ )+ else:+ logger.warning(f"Could not fetch pipeline data for {pipeline_id}")+ pipeline_name = "Unknown API"+ pipeline_type = PipelineType.API.value++ # Use unified status determination with timeout detection+ aggregated_results, execution_status, expected_files = (+ _determine_execution_status_unified(+ file_batch_results=file_batch_results,+ api_client=api_client,+ execution_id=execution_id,+ organization_id=organization_id,+ )+ )++ # Update workflow execution status using unified function+ execution_update_result = _update_execution_status_unified(api_client=api_client,execution_id=execution_id,+ final_status=execution_status,+ aggregated_results=aggregated_results,organization_id=organization_id,)- )- # Update workflow execution status using unified function- execution_update_result = _update_execution_status_unified(- api_client=api_client,- execution_id=execution_id,- final_status=execution_status,- aggregated_results=aggregated_results,- organization_id=organization_id,- )-- # Create minimal context for unified pipeline handling- context = CallbackContext()- context.pipeline_id = pipeline_id- context.execution_id = execution_id- context.organization_id = organization_id- context.workflow_id = workflow_id- context.pipeline_name = pipeline_name- context.pipeline_type = pipeline_type- context.api_client = api_client- context.file_executions = execution_context.get("file_executions", [])- context.pipeline_data = {- "is_api": True- } # Mark as API execution for cleanup logic-- # Add missing UI logs for cost and final workflow status (matching backend behavior)- _publish_final_workflow_ui_logs_api(- context=context,- aggregated_results=aggregated_results,- execution_status=execution_status,- )-- # Handle resource cleanup (matching ETL workflow behavior)- cleanup_result = _cleanup_execution_resources(context)-- # Handle pipeline updates (only if pipeline_id exists)- if context.pipeline_id:- pipeline_result = _handle_pipeline_updates_unified(- context=context, final_status=execution_status, is_api_deployment=True+ # Create minimal context for unified pipeline handling+ context = CallbackContext()+ context.pipeline_id = pipeline_id+ context.execution_id = execution_id+ context.organization_id = organization_id+ context.workflow_id = workflow_id+ context.pipeline_name = pipeline_name+ context.pipeline_type = pipeline_type+ context.api_client = api_client+ context.file_executions = execution_context.get("file_executions", [])+ context.pipeline_data = {+ "is_api": True+ } # Mark as API execution for cleanup logic++ # Add missing UI logs for cost and final workflow status (matching backend behavior)+ _publish_final_workflow_ui_logs_api(+ context=context,+ aggregated_results=aggregated_results,+ execution_status=execution_status,)- else:- logger.info(- f"Skipping pipeline status update for execution {execution_id} "- f"since no APIDeployment record"- )- pipeline_result = {- "skipped": True,- "reason": "execute without APIDeployment record",- }- # Track subscription usage if plugin is present- subscription_tracking_result = _track_subscription_usage_if_available(- context=context,- execution_status=execution_status,- )+ # Handle resource cleanup (matching ETL workflow behavior)+ cleanup_result = _cleanup_execution_resources(context)- # Handle notifications using unified function- notification_result = _handle_notifications_unified(- api_client=api_client,- status=execution_status,- organization_id=organization_id,- execution_id=execution_id,- pipeline_id=pipeline_id,- workflow_id=workflow_id,- pipeline_name=pipeline_name,- pipeline_type=pipeline_type,- error_message=None,- )+ # Handle pipeline updates (only if pipeline_id exists)+ if context.pipeline_id:+ pipeline_result = _handle_pipeline_updates_unified(+ context=context,+ final_status=execution_status,+ is_api_deployment=True,+ )+ else:+ logger.info(+ f"Skipping pipeline status update for execution {execution_id} "+ f"since no APIDeployment record"+ )+ pipeline_result = {+ "skipped": True,+ "reason": "execute without APIDeployment record",+ }++ # Track subscription usage if plugin is present+ subscription_tracking_result = _track_subscription_usage_if_available(+ context=context,+ execution_status=execution_status,+ )- callback_result = {- "execution_id": execution_id,- "workflow_id": workflow_id,- "pipeline_id": pipeline_id,- "status": "completed",- "total_files_processed": aggregated_results.get(- "total_files_processed", 0- ),- "total_execution_time": aggregated_results.get("total_execution_time", 0),- "batches_processed": len(file_batch_results),- "task_id": task_id,- "expected_files": expected_files, # Include expected files for debugging- "execution_update": execution_update_result,- "pipeline_update": pipeline_result,- "notifications": notification_result,- "subscription_tracking_result": subscription_tracking_result,- "cleanup_result": cleanup_result,- "optimization": {- "method": "unified_callback_functions",- "eliminated_code_duplication": True,- "shared_timeout_detection": True,- },- }+ # Handle notifications using unified function+ notification_result = _handle_notifications_unified(+ api_client=api_client,+ status=execution_status,+ organization_id=organization_id,+ execution_id=execution_id,+ pipeline_id=pipeline_id,+ workflow_id=workflow_id,+ pipeline_name=pipeline_name,+ pipeline_type=pipeline_type,+ error_message=None,+ )- logger.info(- f"Successfully completed API callback for execution {execution_id}"- )- return callback_result+ callback_result = {+ "execution_id": execution_id,+ "workflow_id": workflow_id,+ "pipeline_id": pipeline_id,+ "status": "completed",+ "total_files_processed": aggregated_results.get(+ "total_files_processed", 0+ ),+ "total_execution_time": aggregated_results.get(+ "total_execution_time", 0+ ),+ "batches_processed": len(file_batch_results),+ "task_id": task_id,+ "expected_files": expected_files, # Include expected files for debugging+ "execution_update": execution_update_result,+ "pipeline_update": pipeline_result,+ "notifications": notification_result,+ "subscription_tracking_result": subscription_tracking_result,+ "cleanup_result": cleanup_result,+ "optimization": {+ "method": "unified_callback_functions",+ "eliminated_code_duplication": True,+ "shared_timeout_detection": True,+ },+ }- except Exception as e:- logger.error(- f"API callback processing failed for execution {execution_id}: {e}"- )+ logger.info(+ f"Successfully completed API callback for execution {execution_id}"+ )+ return callback_result- # Try to update execution status to failed- try:- # Create organization-scoped API client for error handling- api_client = create_api_client(schema_name)- # Update execution status to error- api_client.update_workflow_execution_status(- execution_id=execution_id,- status=ExecutionStatus.ERROR.value,- error_message=str(e)[:500], # Limit error message length- organization_id=schema_name,+ except Exception as e:+ logger.error(+ f"API callback processing failed for execution {execution_id}: {e}")- # OPTIMIZATION: Skip pipeline status update for API deployments on error- if pipeline_id:- logger.info(- f"OPTIMIZATION: Skipping pipeline status update for API deployment {pipeline_id} on error (no Pipeline record exists)"+ # Try to update execution status to failed (reuse existing api_client)+ try:+ api_client.update_workflow_execution_status(+ execution_id=execution_id,+ status=ExecutionStatus.ERROR.value,+ error_message=str(e)[:500], # Limit error message length+ organization_id=schema_name,)- except Exception as update_error:- logger.error(f"Failed to update execution status: {update_error}")- raise+ # OPTIMIZATION: Skip pipeline status update for API deployments on error+ if pipeline_id:+ logger.info(+ f"OPTIMIZATION: Skipping pipeline status update for API deployment {pipeline_id} on error (no Pipeline record exists)"+ )+ except Exception as update_error:+ logger.error(f"Failed to update execution status: {update_error}")++ raise+ finally:+ try:+ api_client.close()+ except Exception as e:+ logger.warning("api_client.close() failed during cleanup: %s", e)def _publish_final_workflow_ui_logs(diff --git a/workers/conftest.py b/workers/conftest.pynew file mode 100644index 000000000..137e8304e--- /dev/null+++ b/workers/conftest.py@@ -0,0 +1,18 @@+"""Root conftest for workers test suite.++Sets required environment variables before any workers modules are imported.+"""++import os++# These must be set before any workers module import because+# shared/constants/api_endpoints.py evaluates INTERNAL_API_BASE_URL at class definition time.+os.environ.setdefault("INTERNAL_API_BASE_URL", "http://test-backend:8000/internal")+os.environ.setdefault("INTERNAL_SERVICE_API_KEY", "test-key-123")+os.environ.setdefault("CELERY_BROKER_BASE_URL", "amqp://localhost:5672//")+os.environ.setdefault("CELERY_BROKER_USER", "guest")+os.environ.setdefault("CELERY_BROKER_PASS", "guest")+os.environ.setdefault("DB_HOST", "localhost")+os.environ.setdefault("DB_USER", "test")+os.environ.setdefault("DB_PASSWORD", "test")+os.environ.setdefault("DB_NAME", "testdb")diff --git a/workers/sample.env b/workers/sample.envindex a2c10a338..22af85255 100644--- a/workers/sample.env+++ b/workers/sample.env@@ -112,8 +112,14 @@ ENABLE_API_CLIENT_SINGLETON=falseDEBUG_API_CLIENT_INIT=falseWORKER_INFRASTRUCTURE_HEALTH_CHECK=true-# API Client Configuration-API_CLIENT_POOL_SIZE=3+# API Client Connection Pool Size+# Controls pool_connections (number of host connection pools) and pool_maxsize (max connections per pool).+# pool_connections = API_CLIENT_POOL_SIZE, pool_maxsize = API_CLIENT_POOL_SIZE * 2+API_CLIENT_POOL_SIZE=10++# Session lifecycle safety valve - reset singleton after N tasks (0=disabled)+# Only effective when ENABLE_API_CLIENT_SINGLETON=true+WORKER_SINGLETON_RESET_THRESHOLD=1000# Config CachingENABLE_CONFIG_CACHE=truediff --git a/workers/shared/api/internal_client.py b/workers/shared/api/internal_client.pyindex dc19db881..def90bd86 100644--- a/workers/shared/api/internal_client.py+++ b/workers/shared/api/internal_client.py@@ -10,6 +10,8 @@ unified interface."""import logging+import threading+import timeimport uuidfrom typing import Anyfrom uuid import UUID@@ -85,6 +87,12 @@ class InternalAPIClient(CachedAPIClientMixin):_shared_session = None_initialization_count = 0+ # Task counter for periodic singleton reset (FR-3.2)+ _task_counter: int = 0+ _task_counter_lock: threading.Lock = threading.Lock()+ _last_reset_time: float | None = None+ _cached_reset_threshold: int | None = None+def __init__(self, config: WorkerConfig | None = None):"""Initialize the facade with all specialized clients and caching.@@ -126,6 +134,8 @@ class InternalAPIClient(CachedAPIClientMixin):logger.info("Creating shared HTTP session (GIL-safe singleton pattern)")# Create the first base client to establish the sessionself.base_client = BaseAPIClient(self.config)+ # The first client owns the session; reset_singleton() handles final cleanup+ self.base_client._owns_session = False# Share the session for reuse (atomic assignment)InternalAPIClient._shared_session = self.base_client.sessionInternalAPIClient._shared_base_client = self.base_client@@ -140,35 +150,35 @@ class InternalAPIClient(CachedAPIClientMixin):self.base_client.session = (InternalAPIClient._shared_session) # Use shared session+ self.base_client._owns_session = False++ # Helper to share session on a client+ def _share_session(client: BaseAPIClient) -> None:+ client.session.close() # Close the freshly-created session+ client.session = InternalAPIClient._shared_session+ client._owns_session = False# Create specialized clients with shared session (outside lock for performance)self.execution_client = ExecutionAPIClient(self.config)- self.execution_client.session.close()- self.execution_client.session = InternalAPIClient._shared_session+ _share_session(self.execution_client)self.workflow_client = WorkflowAPIClient()- self.workflow_client.session.close()- self.workflow_client.session = InternalAPIClient._shared_session+ _share_session(self.workflow_client)self.file_client = FileAPIClient(self.config)- self.file_client.session.close()- self.file_client.session = InternalAPIClient._shared_session+ _share_session(self.file_client)self.webhook_client = WebhookAPIClient(self.config)- self.webhook_client.session.close()- self.webhook_client.session = InternalAPIClient._shared_session+ _share_session(self.webhook_client)self.organization_client = OrganizationAPIClient(self.config)- self.organization_client.session.close()- self.organization_client.session = InternalAPIClient._shared_session+ _share_session(self.organization_client)self.tool_client = ToolAPIClient(self.config)- self.tool_client.session.close()- self.tool_client.session = InternalAPIClient._shared_session+ _share_session(self.tool_client)self.usage_client = UsageAPIClient(self.config)- self.usage_client.session.close()- self.usage_client.session = InternalAPIClient._shared_session+ _share_session(self.usage_client)def _initialize_core_clients_traditional(self) -> None:"""Initialize clients the traditional way (for backward compatibility)."""@@ -247,6 +257,62 @@ class InternalAPIClient(CachedAPIClientMixin):self.usage_client.close()logger.debug("Closed all InternalAPIClient sessions (traditional mode)")+ @classmethod+ def reset_singleton(cls):+ """Reset shared singleton session state. Safe to call anytime.++ Note: In thread/gevent/eventlet pools, this may close the session while+ other threads have in-flight requests. The BaseAPIClient retry logic+ handles transient connection errors from this race. For prefork pools,+ this is a non-issue (one task per process).+ """+ if cls._shared_session is not None:+ try:+ cls._shared_session.close()+ except Exception as e:+ logger.warning("Failed to close shared session during reset: %s", e)+ cls._shared_session = None+ cls._shared_base_client = None+ cls._initialization_count = 0+ cls._task_counter = 0+ cls._cached_reset_threshold = None+ logger.info("Reset InternalAPIClient singleton state")++ @classmethod+ def increment_task_counter(cls) -> None:+ """Increment task counter and reset singleton if threshold reached.++ Called via task_postrun signal after each task completes.+ Uses a lock for thread safety in case threads/gevent/eventlet pools are used.+ When singleton disabled (default), reset_singleton() is a no-op.+ """+ if cls._cached_reset_threshold is None:+ cls._cached_reset_threshold = WorkerConfig().singleton_reset_task_threshold++ with cls._task_counter_lock:+ cls._task_counter += 1+ if (+ cls._cached_reset_threshold > 0+ and cls._task_counter >= cls._cached_reset_threshold+ ):+ logger.info(+ "Task counter reached threshold (%d/%d), resetting singleton session",+ cls._task_counter,+ cls._cached_reset_threshold,+ )+ cls.reset_singleton()+ cls._last_reset_time = time.time()++ @classmethod+ def get_task_counter_info(cls) -> dict:+ """Get task counter state for observability."""+ return {+ "task_counter": cls._task_counter,+ "last_reset_time": cls._last_reset_time,+ "shared_session_active": cls._shared_session is not None,+ "initialization_count": cls._initialization_count,+ }+def __exit__(self, exc_type, exc_val, exc_tb):"""Context manager exit with safe cleanup."""self.close()diff --git a/workers/shared/clients/base_client.py b/workers/shared/clients/base_client.pyindex 69759cd3a..19961938f 100644--- a/workers/shared/clients/base_client.py+++ b/workers/shared/clients/base_client.py@@ -103,8 +103,12 @@ class BaseAPIClient:# It comes from task context, not from configurationself.organization_id = None+ # Track whether this client owns its session (for singleton-aware close)+ self._owns_session = True+# Initialize requests session with retry strategyself.session = requests.Session()+ self._closed = False # Track session state for idempotent closeself._setup_session()# Always log initialization@@ -152,10 +156,13 @@ class BaseAPIClient:)# HTTP adapter with connection pooling+ # pool_connections: number of connection pools to cache (one per host:port)+ # pool_maxsize: max connections per pool (concurrent requests per host)+ pool_size = self.config.api_client_pool_sizeadapter = HTTPAdapter(max_retries=retry_strategy,- pool_connections=10, # Number of connection pools- pool_maxsize=20, # Maximum number of connections per pool+ pool_connections=pool_size,+ pool_maxsize=pool_size * 2,pool_block=False, # Don't block when pool is full)self.session.mount("http://", adapter)@@ -551,8 +558,19 @@ class BaseAPIClient:# Session managementdef close(self):- """Close the HTTP session."""+ """Close the HTTP session (idempotent).++ When _owns_session is False (singleton mode), this is a no-op to avoid+ closing the shared session that other clients depend on.+ """+ if self._closed:+ return+ if not self._owns_session:+ logger.debug("Skipping session close (shared singleton session)")+ self._closed = True+ returnself.session.close()+ self._closed = Truelogger.debug("Closed API client session")def __enter__(self):@@ -562,3 +580,17 @@ class BaseAPIClient:def __exit__(self, exc_type, exc_val, exc_tb):"""Context manager exit."""self.close()++ def __del__(self):+ """Safety net: close session on garbage collection.++ Only closes if this client owns its session (not shared singleton).+ """+ try:+ if hasattr(self, "_closed") and not self._closed:+ if hasattr(self, "_owns_session") and not self._owns_session:+ return # Don't close shared singleton session+ if hasattr(self, "session") and self.session is not None:+ self.session.close()+ except Exception:+ pass # Never raise in __del__diff --git a/workers/shared/infrastructure/config/worker_config.py b/workers/shared/infrastructure/config/worker_config.pyindex eb1c0b8e7..e1e11997c 100644--- a/workers/shared/infrastructure/config/worker_config.py+++ b/workers/shared/infrastructure/config/worker_config.py@@ -257,7 +257,14 @@ class WorkerConfig:== "true")api_client_pool_size: int = field(- default_factory=lambda: int(os.getenv("API_CLIENT_POOL_SIZE", "3"))+ default_factory=lambda: int(os.getenv("API_CLIENT_POOL_SIZE", "10"))+ )++ # Session Lifecycle Safety Valve (FR-3.2)+ # Reset singleton session after N task completions to prevent FD accumulation.+ # Only effective when enable_api_client_singleton=true. Set to 0 to disable.+ singleton_reset_task_threshold: int = field(+ default_factory=lambda: int(os.getenv("WORKER_SINGLETON_RESET_THRESHOLD", "1000")))# Configuration Cachingdiff --git a/workers/shared/patterns/factory/__init__.py b/workers/shared/patterns/factory/__init__.pyindex ef61e95d1..bce87ef3e 100644--- a/workers/shared/patterns/factory/__init__.py+++ b/workers/shared/patterns/factory/__init__.py@@ -4,6 +4,6 @@ This package provides factory implementations following the Factorypattern for creating various worker components."""-from .client_factory import InternalAPIClientFactory+from .client_factory import ClientFactory-__all__ = ["InternalAPIClientFactory"]+__all__ = ["ClientFactory"]diff --git a/workers/shared/patterns/factory/client_factory.py b/workers/shared/patterns/factory/client_factory.pyindex 419a494d8..8ed5311b6 100644--- a/workers/shared/patterns/factory/client_factory.py+++ b/workers/shared/patterns/factory/client_factory.py@@ -5,7 +5,7 @@ performance optimizations while maintaining readability and testability."""from threading import Lock-from typing import TypeVar+from typing import Any, TypeVarfrom ...clients.base_client import BaseAPIClientfrom ...clients.execution_client import ExecutionAPIClient@@ -162,7 +162,7 @@ class CachingConfigurationMixin:def __init__(self, *args, **kwargs):super().__init__(*args, **kwargs)- self._cache: dict[str, any] = {}+ self._cache: dict[str, Any] = {}self._cache_timestamps: dict[str, float] = {}self._cache_lock = Lock()@@ -179,12 +179,12 @@ class CachingConfigurationMixin:return (time.time() - self._cache_timestamps[cache_key]) < ttl- def _get_from_cache(self, cache_key: str) -> any | None:+ def _get_from_cache(self, cache_key: str) -> Any | None:"""Thread-safe cache retrieval."""with self._cache_lock:return self._cache.get(cache_key)- def _set_cache(self, cache_key: str, value: any) -> None:+ def _set_cache(self, cache_key: str, value: Any) -> None:"""Thread-safe cache storage."""import timediff --git a/workers/shared/tests/__init__.py b/workers/shared/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/workers/shared/tests/test_session_lifecycle.py b/workers/shared/tests/test_session_lifecycle.pynew file mode 100644index 000000000..aa482ba9e--- /dev/null+++ b/workers/shared/tests/test_session_lifecycle.py@@ -0,0 +1,550 @@+"""Tests for HTTP Session Lifecycle Management (UNS-205).++Tests cover:+- FR-1: __del__ destructor safety net+- FR-2: Explicit cleanup in callback tasks (try/finally)+- FR-3: Singleton lifecycle management (reset_singleton, task counter)+- Gap #5: _owns_session flag for singleton-safe close()+- Gap #4: API_CLIENT_POOL_SIZE wired into HTTPAdapter+"""++from unittest.mock import MagicMock, patch++import pytest+++# ---------------------------------------------------------------------------+# Fixtures+# ---------------------------------------------------------------------------++[email protected](autouse=True)+def _reset_singleton_state():+ """Reset InternalAPIClient class-level singleton state between tests."""+ from shared.api.internal_client import InternalAPIClient++ InternalAPIClient._shared_session = None+ InternalAPIClient._shared_base_client = None+ InternalAPIClient._initialization_count = 0+ InternalAPIClient._task_counter = 0+ InternalAPIClient._last_reset_time = None+ InternalAPIClient._cached_reset_threshold = None+ yield+ InternalAPIClient._shared_session = None+ InternalAPIClient._shared_base_client = None+ InternalAPIClient._initialization_count = 0+ InternalAPIClient._task_counter = 0+ InternalAPIClient._last_reset_time = None+ InternalAPIClient._cached_reset_threshold = None+++def mock_config():+ """Create a WorkerConfig with defaults suitable for testing."""+ with patch.dict(+ "os.environ",+ {+ "INTERNAL_API_BASE_URL": "http://test-backend:8000/internal",+ "INTERNAL_SERVICE_API_KEY": "test-key-123",+ "CELERY_BROKER_BASE_URL": "amqp://localhost:5672//",+ "CELERY_BROKER_USER": "guest",+ "CELERY_BROKER_PASS": "guest",+ "DB_HOST": "localhost",+ "DB_USER": "test",+ "DB_PASSWORD": "test",+ "DB_NAME": "testdb",+ "API_CLIENT_POOL_SIZE": "5",+ "ENABLE_API_CLIENT_SINGLETON": "false",+ },+ ):+ from shared.infrastructure.config.worker_config import WorkerConfig++ yield WorkerConfig()+++def mock_config_singleton():+ """Create a WorkerConfig with singleton enabled."""+ with patch.dict(+ "os.environ",+ {+ "INTERNAL_API_BASE_URL": "http://test-backend:8000/internal",+ "INTERNAL_SERVICE_API_KEY": "test-key-123",+ "CELERY_BROKER_BASE_URL": "amqp://localhost:5672//",+ "CELERY_BROKER_USER": "guest",+ "CELERY_BROKER_PASS": "guest",+ "DB_HOST": "localhost",+ "DB_USER": "test",+ "DB_PASSWORD": "test",+ "DB_NAME": "testdb",+ "API_CLIENT_POOL_SIZE": "5",+ "ENABLE_API_CLIENT_SINGLETON": "true",+ "WORKER_SINGLETON_RESET_THRESHOLD": "3",+ },+ ):+ from shared.infrastructure.config.worker_config import WorkerConfig++ yield WorkerConfig()+++# ===========================================================================+# FR-1: __del__ destructor tests+# ===========================================================================+++class TestBaseAPIClientDestructor:+ """Tests for BaseAPIClient.__del__ safety net."""++ def test_del_closes_unclosed_session(self, mock_config):+ """__del__ should close the session if close() was never called."""+ from shared.clients.base_client import BaseAPIClient++ client = BaseAPIClient(mock_config)+ mock_session = MagicMock()+ client.session = mock_session+ client._closed = False+ client._owns_session = True++ client.__del__()++ mock_session.close.assert_called_once()++ def test_del_skips_already_closed_session(self, mock_config):+ """__del__ should be a no-op if session is already closed."""+ from shared.clients.base_client import BaseAPIClient++ client = BaseAPIClient(mock_config)+ mock_session = MagicMock()+ client.session = mock_session+ client._closed = True++ client.__del__()++ mock_session.close.assert_not_called()++ def test_del_skips_shared_session(self, mock_config):+ """__del__ should NOT close a shared singleton session."""+ from shared.clients.base_client import BaseAPIClient++ client = BaseAPIClient(mock_config)+ mock_session = MagicMock()+ client.session = mock_session+ client._closed = False+ client._owns_session = False++ client.__del__()++ mock_session.close.assert_not_called()++ def test_del_handles_missing_attributes(self):+ """__del__ should not raise even if init failed partially."""+ from shared.clients.base_client import BaseAPIClient++ client = object.__new__(BaseAPIClient)+ # No attributes set at all - should not raise+ client.__del__()++ def test_del_swallows_exceptions(self, mock_config):+ """__del__ should never propagate exceptions."""+ from shared.clients.base_client import BaseAPIClient++ client = BaseAPIClient(mock_config)+ client._closed = False+ client._owns_session = True+ client.session = MagicMock()+ client.session.close.side_effect = RuntimeError("connection broken")++ # Should not raise+ client.__del__()+++# ===========================================================================+# FR-1 + Gap #5: close() with _owns_session+# ===========================================================================+++class TestBaseAPIClientClose:+ """Tests for BaseAPIClient.close() behavior."""++ def test_close_is_idempotent(self, mock_config):+ """Calling close() multiple times should only close session once."""+ from shared.clients.base_client import BaseAPIClient++ client = BaseAPIClient(mock_config)+ mock_session = MagicMock()+ client.session = mock_session++ client.close()+ client.close()+ client.close()++ mock_session.close.assert_called_once()++ def test_close_skips_shared_session(self, mock_config):+ """close() should NOT close the session when _owns_session=False."""+ from shared.clients.base_client import BaseAPIClient++ client = BaseAPIClient(mock_config)+ mock_session = MagicMock()+ client.session = mock_session+ client._owns_session = False++ client.close()++ mock_session.close.assert_not_called()+ assert client._closed is True # Flag still set to prevent redundant calls++ def test_close_closes_owned_session(self, mock_config):+ """close() should close the session when _owns_session=True (default)."""+ from shared.clients.base_client import BaseAPIClient++ client = BaseAPIClient(mock_config)+ mock_session = MagicMock()+ client.session = mock_session+ assert client._owns_session is True # Default++ client.close()++ mock_session.close.assert_called_once()+ assert client._closed is True++ def test_context_manager_calls_close(self, mock_config):+ """Using 'with' should call close() on exit."""+ from shared.clients.base_client import BaseAPIClient++ with BaseAPIClient(mock_config) as client:+ mock_session = MagicMock()+ client.session = mock_session++ mock_session.close.assert_called_once()+++# ===========================================================================+# Gap #4: API_CLIENT_POOL_SIZE wired into HTTPAdapter+# ===========================================================================+++class TestPoolSizeConfiguration:+ """Tests for API_CLIENT_POOL_SIZE being wired into HTTPAdapter."""++ def test_pool_size_from_config(self, mock_config):+ """HTTPAdapter should use api_client_pool_size from config."""+ from shared.clients.base_client import BaseAPIClient++ assert mock_config.api_client_pool_size == 5++ client = BaseAPIClient(mock_config)++ # Inspect the mounted adapter's internal pool settings+ adapter = client.session.get_adapter("http://")+ assert adapter._pool_connections == 5+ assert adapter._pool_maxsize == 10 # pool_size * 2++ client.close()++ def test_default_pool_size(self):+ """Default pool size should be 10 when not configured."""+ with patch.dict(+ "os.environ",+ {+ "INTERNAL_API_BASE_URL": "http://test:8000/internal",+ "INTERNAL_SERVICE_API_KEY": "test-key",+ "CELERY_BROKER_BASE_URL": "amqp://localhost:5672//",+ "CELERY_BROKER_USER": "guest",+ "CELERY_BROKER_PASS": "guest",+ "DB_HOST": "localhost",+ "DB_USER": "test",+ "DB_PASSWORD": "test",+ "DB_NAME": "testdb",+ },+ clear=False,+ ):+ from shared.infrastructure.config.worker_config import WorkerConfig++ config = WorkerConfig()+ assert config.api_client_pool_size == 10+++# ===========================================================================+# FR-3: Singleton lifecycle management+# ===========================================================================+++class TestResetSingleton:+ """Tests for InternalAPIClient.reset_singleton()."""++ def test_reset_when_no_shared_session(self):+ """reset_singleton() should be a no-op when there's no shared session."""+ from shared.api.internal_client import InternalAPIClient++ # Should not raise+ InternalAPIClient.reset_singleton()+ assert InternalAPIClient._shared_session is None++ def test_reset_closes_shared_session(self):+ """reset_singleton() should close the shared session and clear state."""+ from shared.api.internal_client import InternalAPIClient++ mock_session = MagicMock()+ InternalAPIClient._shared_session = mock_session+ InternalAPIClient._shared_base_client = MagicMock()+ InternalAPIClient._initialization_count = 5+ InternalAPIClient._task_counter = 42++ InternalAPIClient.reset_singleton()++ mock_session.close.assert_called_once()+ assert InternalAPIClient._shared_session is None+ assert InternalAPIClient._shared_base_client is None+ assert InternalAPIClient._initialization_count == 0+ assert InternalAPIClient._task_counter == 0+ assert InternalAPIClient._cached_reset_threshold is None++ def test_reset_handles_close_exception(self):+ """reset_singleton() should handle session.close() failure gracefully."""+ from shared.api.internal_client import InternalAPIClient++ mock_session = MagicMock()+ mock_session.close.side_effect = RuntimeError("broken pipe")+ InternalAPIClient._shared_session = mock_session++ # Should not raise+ InternalAPIClient.reset_singleton()++ assert InternalAPIClient._shared_session is None+++# ===========================================================================+# FR-3: Task counter+# ===========================================================================+++class TestTaskCounter:+ """Tests for InternalAPIClient.increment_task_counter()."""++ def test_increment_counter(self, mock_config_singleton):+ """Counter should increment on each call."""+ from shared.api.internal_client import InternalAPIClient++ InternalAPIClient._task_counter = 0++ with patch.object(+ InternalAPIClient, "reset_singleton"+ ) as mock_reset:+ InternalAPIClient.increment_task_counter()+ assert InternalAPIClient._task_counter == 1+ mock_reset.assert_not_called()++ def test_threshold_triggers_reset(self, mock_config_singleton):+ """Counter reaching threshold should trigger reset_singleton()."""+ from shared.api.internal_client import InternalAPIClient++ # Threshold is 3 from mock_config_singleton+ InternalAPIClient._task_counter = 2 # One away from threshold++ with patch.object(+ InternalAPIClient, "reset_singleton"+ ) as mock_reset:+ InternalAPIClient.increment_task_counter()+ mock_reset.assert_called_once()+ # Counter zeroing is done inside reset_singleton() itself,+ # which is mocked here — verified in TestResetSingleton instead++ def test_get_task_counter_info(self):+ """get_task_counter_info() should return correct state."""+ from shared.api.internal_client import InternalAPIClient++ InternalAPIClient._task_counter = 42+ InternalAPIClient._shared_session = MagicMock()++ info = InternalAPIClient.get_task_counter_info()++ assert info["task_counter"] == 42+ assert info["shared_session_active"] is True+++# ===========================================================================+# FR-3: on_task_postrun guard+# ===========================================================================+++class TestOnTaskPostrunGuard:+ """Tests for the singleton guard in on_task_postrun."""++ def test_postrun_skips_when_singleton_disabled(self):+ """on_task_postrun should skip increment when singleton=false."""+ import worker++ with patch.object(worker, "config") as mock_cfg:+ mock_cfg.enable_api_client_singleton = False+ with patch(+ "shared.api.internal_client.InternalAPIClient"+ ".increment_task_counter"+ ) as mock_increment:+ worker.on_task_postrun(sender=None, task_id=None)+ mock_increment.assert_not_called()++ def test_postrun_calls_increment_when_singleton_enabled(self):+ """on_task_postrun should call increment when singleton=true."""+ import worker++ with patch.object(worker, "config") as mock_cfg:+ mock_cfg.enable_api_client_singleton = True+ with patch(+ "shared.api.internal_client.InternalAPIClient"+ ".increment_task_counter"+ ) as mock_increment:+ worker.on_task_postrun(sender=None, task_id=None)+ mock_increment.assert_called_once()+++# ===========================================================================+# Gap #5: Singleton-aware close() in InternalAPIClient+# ===========================================================================+++class TestInternalAPIClientSingletonClose:+ """Tests for InternalAPIClient.close() respecting singleton mode."""++ def test_close_traditional_mode_closes_all(self, mock_config):+ """In traditional mode, close() should close all client sessions."""+ from shared.api.internal_client import InternalAPIClient++ with patch("shared.api.internal_client.get_client_plugin", return_value=None):+ client = InternalAPIClient(mock_config)++ # Replace sessions with mocks+ mock_sessions = {}+ for attr in [+ "base_client",+ "execution_client",+ "file_client",+ "webhook_client",+ "organization_client",+ "tool_client",+ "workflow_client",+ "usage_client",+ ]:+ sub_client = getattr(client, attr)+ mock_session = MagicMock()+ sub_client.session = mock_session+ sub_client._closed = False+ sub_client._owns_session = True+ mock_sessions[attr] = mock_session++ client.close()++ for attr, mock_session in mock_sessions.items():+ mock_session.close.assert_called_once(), (+ f"{attr} session was not closed"+ )++ def test_close_singleton_mode_preserves_shared_session(+ self, mock_config_singleton+ ):+ """In singleton mode, close() should NOT close the shared session."""+ from shared.api.internal_client import InternalAPIClient++ with patch("shared.api.internal_client.get_client_plugin", return_value=None):+ client = InternalAPIClient(mock_config_singleton)++ shared_session = InternalAPIClient._shared_session+ assert shared_session is not None++ # close() in singleton mode should preserve the session+ client.close()++ # The shared session should still be alive+ assert InternalAPIClient._shared_session is shared_session++ def test_singleton_clients_have_owns_session_false(+ self, mock_config_singleton+ ):+ """All clients in singleton mode should have _owns_session=False."""+ from shared.api.internal_client import InternalAPIClient++ with patch("shared.api.internal_client.get_client_plugin", return_value=None):+ client = InternalAPIClient(mock_config_singleton)++ for attr in [+ "base_client",+ "execution_client",+ "file_client",+ "webhook_client",+ "organization_client",+ "tool_client",+ "workflow_client",+ "usage_client",+ ]:+ sub_client = getattr(client, attr)+ assert sub_client._owns_session is False, (+ f"{attr} should have _owns_session=False in singleton mode"+ )++ def test_traditional_clients_have_owns_session_true(self, mock_config):+ """All clients in traditional mode should have _owns_session=True."""+ from shared.api.internal_client import InternalAPIClient++ with patch("shared.api.internal_client.get_client_plugin", return_value=None):+ client = InternalAPIClient(mock_config)++ for attr in [+ "base_client",+ "execution_client",+ "file_client",+ "webhook_client",+ "organization_client",+ "tool_client",+ "workflow_client",+ "usage_client",+ ]:+ sub_client = getattr(client, attr)+ assert sub_client._owns_session is True, (+ f"{attr} should have _owns_session=True in traditional mode"+ )+++# ===========================================================================+# FR-2: WorkerExecutionContext managed_execution_context cleanup+# ===========================================================================+++class TestManagedExecutionContextCleanup:+ """Tests for context manager cleanup in WorkerExecutionContext."""++ def test_managed_context_closes_client_on_success(self, mock_config):+ """managed_execution_context should close client after successful use."""+ from shared.workflow.execution.context import WorkerExecutionContext++ with patch.object(+ WorkerExecutionContext,+ "setup_execution_context",+ ) as mock_setup:+ mock_client = MagicMock()+ mock_setup.return_value = (mock_config, mock_client)++ with WorkerExecutionContext.managed_execution_context(+ "org-1", "exec-1", "wf-1"+ ) as (cfg, client):+ pass # Simulate successful execution++ mock_client.close.assert_called_once()++ def test_managed_context_closes_client_on_exception(self, mock_config):+ """managed_execution_context should close client even when exception occurs."""+ from shared.workflow.execution.context import WorkerExecutionContext++ with patch.object(+ WorkerExecutionContext,+ "setup_execution_context",+ ) as mock_setup:+ mock_client = MagicMock()+ mock_setup.return_value = (mock_config, mock_client)++ with pytest.raises(ValueError):+ with WorkerExecutionContext.managed_execution_context(+ "org-1", "exec-1", "wf-1"+ ) as (cfg, client):+ raise ValueError("test error")++ mock_client.close.assert_called_once()diff --git a/workers/worker.py b/workers/worker.pyindex aa3d6b600..ca5dce202 100755--- a/workers/worker.py+++ b/workers/worker.py@@ -23,10 +23,12 @@ from celery import bootsteps, signals # noqa: E402sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))# Import the WorkerBuilder and WorkerType+from shared.api.internal_client import InternalAPIClient # noqa: E402from shared.enums.worker_enums import WorkerType # noqa: E402from shared.infrastructure import initialize_worker_infrastructure # noqa: E402from shared.infrastructure.config.builder import WorkerBuilder # noqa: E402from shared.models.worker_models import get_celery_setting # noqa: E402+from shared.patterns.factory.client_factory import ClientFactory # noqa: E402# Determine worker type from environment FIRSTWORKER_TYPE = os.environ.get("WORKER_TYPE", "general")@@ -366,6 +368,44 @@ def on_worker_process_init(**kwargs):# ============= END OF WORKER PROCESS INIT HOOK =============+# ============= WORKER PROCESS SHUTDOWN HOOK =============[email protected]_process_shutdown.connect+def on_worker_process_shutdown(**kwargs):+ """Clean up HTTP sessions during worker shutdown."""+ logger.info("Cleaning up API client resources (PID: %s)", os.getpid())+ try:+ InternalAPIClient.reset_singleton()+ except Exception as e:+ logger.warning(f"Failed to reset InternalAPIClient singleton: {e}")+ try:+ ClientFactory.reset_shared_state()+ except Exception as e:+ logger.warning(f"Failed to reset ClientFactory: {e}")+++# ============= END OF WORKER PROCESS SHUTDOWN HOOK =============+++# ============= TASK COMPLETION HOOK (SESSION LIFECYCLE SAFETY VALVE) =============[email protected]_postrun.connect+def on_task_postrun(sender=None, task_id=None, **kwargs):+ """Increment task counter for periodic singleton session reset (FR-3.2).++ After WORKER_SINGLETON_RESET_THRESHOLD tasks (default: 1000), the singleton+ session is closed and recreated. Skipped when singleton is disabled (default)+ since the counter and reset would be no-ops.+ """+ if not config.enable_api_client_singleton:+ return+ try:+ InternalAPIClient.increment_task_counter()+ except Exception as e:+ logger.warning(f"Failed to increment task counter: {e}")+++# ============= END OF TASK COMPLETION HOOK =============++# ============= REGISTER HEARTBEATKEEPER HERE =============# Check if HeartbeatKeeper should be enabled (default: enabled)# Uses hierarchical config system: CLI args > worker-specific > global > default