Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2942 [HOT-FIX] Fix API deployment directory cleanup and MinIO compatibility (#1637)

ali7ee13642dbac
diff --git a/workers/callback/tasks.py b/workers/callback/tasks.py
index 8dc66aa53..a7496c3d4 100644
--- a/workers/callback/tasks.py
+++ b/workers/callback/tasks.py
@@ -1037,7 +1037,7 @@ def _get_execution_directories(context: CallbackContext) -> list[tuple[str, any,
directories_to_clean = []
- if is_api_execution and context.pipeline_id and context.workflow_id:
+ if is_api_execution and context.workflow_id:
# API execution - clean BOTH API execution directory AND workflow execution directory
# 1. API execution directory
@@ -1047,7 +1047,9 @@ def _get_execution_directories(context: CallbackContext) -> list[tuple[str, any,
execution_id=context.execution_id,
organization_id=context.organization_id,
)
- directories_to_clean.append((api_execution_dir, FileStorageType.API, "api"))
+ directories_to_clean.append(
+ (api_execution_dir, FileStorageType.API_EXECUTION, "api")
+ )
logger.info(f"Added API execution directory for cleanup: {api_execution_dir}")
except Exception as e:
logger.warning(f"Could not get API execution directory: {e}")
@@ -1563,6 +1565,9 @@ def process_batch_callback_api(
context.pipeline_name = pipeline_name
context.pipeline_type = pipeline_type
context.api_client = api_client
+ context.pipeline_data = {
+ "is_api": True
+ } # Mark as API execution for cleanup logic
# Add missing UI logs for cost and final workflow status (matching backend behavior)
_publish_final_workflow_ui_logs_api(
@@ -1571,6 +1576,9 @@ def process_batch_callback_api(
execution_status=execution_status,
)
+ # Handle resource cleanup (matching ETL workflow behavior)
+ cleanup_result = _cleanup_execution_resources(context)
+
# Handle pipeline updates (skip for API deployments)
pipeline_result = _handle_pipeline_updates_unified(
context=context, final_status=execution_status, is_api_deployment=True
@@ -1604,6 +1612,7 @@ def process_batch_callback_api(
"execution_update": execution_update_result,
"pipeline_update": pipeline_result,
"notifications": notification_result,
+ "cleanup_result": cleanup_result,
"optimization": {
"method": "unified_callback_functions",
"eliminated_code_duplication": True,
diff --git a/workers/pyproject.toml b/workers/pyproject.toml
index 98c0522cf..29c3fffcf 100644
--- a/workers/pyproject.toml
+++ b/workers/pyproject.toml
@@ -11,6 +11,8 @@ requires-python = ">=3.12"
dependencies = [
# Worker runtime
"celery>=5.5.3", # Latest stable version - AMQP support built into core
+ # AWS/S3 client - pin to match backend for MinIO compatibility
+ "boto3~=1.34.0", # Pinned to 1.34.x - boto3 1.36+ breaks MinIO DeleteObjects
# HTTP clients and utilities (merged from shared package)
"requests>=2.31.0,<3.0.0", # HTTP client for internal API calls
"urllib3>=1.26.0", # HTTP utilities and retry strategies
diff --git a/workers/uv.lock b/workers/uv.lock
index 84fa4e577..54f59a01d 100644
--- a/workers/uv.lock
+++ b/workers/uv.lock
@@ -4635,6 +4635,7 @@ name = "unstract-workers"
version = "1.0.0"
source = { editable = "." }
dependencies = [
+ { name = "boto3" },
{ name = "celery" },
{ name = "httpx" },
{ name = "prometheus-client" },
@@ -4681,6 +4682,7 @@ test = [
[package.metadata]
requires-dist = [
+ { name = "boto3", specifier = "~=1.34.0" },
{ name = "celery", specifier = ">=5.5.3" },
{ name = "httpx", specifier = ">=0.27.0" },
{ name = "prometheus-client", specifier = ">=0.17.0,<1.0.0" },