LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2942 [HOT-FIX] Fix API deployment directory cleanup and MinIO compatibility (#1637)
ali7ee13642dbac
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.pyindex 8dc66aa53..a7496c3d4 100644--- a/workers/callback/tasks.py+++ b/workers/callback/tasks.py@@ -1037,7 +1037,7 @@ def _get_execution_directories(context: CallbackContext) -> list[tuple[str, any,directories_to_clean = []- if is_api_execution and context.pipeline_id and context.workflow_id:+ if is_api_execution and context.workflow_id:# API execution - clean BOTH API execution directory AND workflow execution directory# 1. API execution directory@@ -1047,7 +1047,9 @@ def _get_execution_directories(context: CallbackContext) -> list[tuple[str, any,execution_id=context.execution_id,organization_id=context.organization_id,)- directories_to_clean.append((api_execution_dir, FileStorageType.API, "api"))+ directories_to_clean.append(+ (api_execution_dir, FileStorageType.API_EXECUTION, "api")+ )logger.info(f"Added API execution directory for cleanup: {api_execution_dir}")except Exception as e:logger.warning(f"Could not get API execution directory: {e}")@@ -1563,6 +1565,9 @@ def process_batch_callback_api(context.pipeline_name = pipeline_namecontext.pipeline_type = pipeline_typecontext.api_client = api_client+ context.pipeline_data = {+ "is_api": True+ } # Mark as API execution for cleanup logic# Add missing UI logs for cost and final workflow status (matching backend behavior)_publish_final_workflow_ui_logs_api(@@ -1571,6 +1576,9 @@ def process_batch_callback_api(execution_status=execution_status,)+ # Handle resource cleanup (matching ETL workflow behavior)+ cleanup_result = _cleanup_execution_resources(context)+# Handle pipeline updates (skip for API deployments)pipeline_result = _handle_pipeline_updates_unified(context=context, final_status=execution_status, is_api_deployment=True@@ -1604,6 +1612,7 @@ def process_batch_callback_api("execution_update": execution_update_result,"pipeline_update": pipeline_result,"notifications": notification_result,+ "cleanup_result": cleanup_result,"optimization": {"method": "unified_callback_functions","eliminated_code_duplication": True,diff --git a/workers/pyproject.toml b/workers/pyproject.tomlindex 98c0522cf..29c3fffcf 100644--- a/workers/pyproject.toml+++ b/workers/pyproject.toml@@ -11,6 +11,8 @@ requires-python = ">=3.12"dependencies = [# Worker runtime"celery>=5.5.3", # Latest stable version - AMQP support built into core+ # AWS/S3 client - pin to match backend for MinIO compatibility+ "boto3~=1.34.0", # Pinned to 1.34.x - boto3 1.36+ breaks MinIO DeleteObjects# HTTP clients and utilities (merged from shared package)"requests>=2.31.0,<3.0.0", # HTTP client for internal API calls"urllib3>=1.26.0", # HTTP utilities and retry strategiesdiff --git a/workers/uv.lock b/workers/uv.lockindex 84fa4e577..54f59a01d 100644--- a/workers/uv.lock+++ b/workers/uv.lock@@ -4635,6 +4635,7 @@ name = "unstract-workers"version = "1.0.0"source = { editable = "." }dependencies = [+ { name = "boto3" },{ name = "celery" },{ name = "httpx" },{ name = "prometheus-client" },@@ -4681,6 +4682,7 @@ test = [[package.metadata]requires-dist = [+ { name = "boto3", specifier = "~=1.34.0" },{ name = "celery", specifier = ">=5.5.3" },{ name = "httpx", specifier = ">=0.27.0" },{ name = "prometheus-client", specifier = ">=0.17.0,<1.0.0" },