LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2663 [FEAT] moving away from redis hitl file handling improvements (#1550)
vishnuszipstack38b3ce1fc2ca
diff --git a/backend/sample.env b/backend/sample.envindex 4001f114c..5413c2c5c 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -231,5 +231,8 @@ RUNNER_POLLING_INTERVAL_SECONDS=2# Examples: 900 (15 min), 1800 (30 min), 3600 (60 min)MIN_SCHEDULE_INTERVAL_SECONDS=1800+# HITL Files Storage Configuration+HITL_FILES_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'+# File active cache redis dbFILE_ACTIVE_CACHE_REDIS_DB=0diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex 44e2ac853..f86f76827 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -290,7 +290,6 @@ class DestinationConnector(BaseConnector):"""Handle the output based on the connection type."""connection_type = self.endpoint.connection_typetool_execution_result: str | None = None-if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:self.copy_output_to_output_directory()@@ -940,18 +939,30 @@ class DestinationConnector(BaseConnector):q_name: Queue name for regular queuettl_seconds: TTL in seconds (optional, for regular queue)"""- # Get queue instance- conn = QueueUtils.get_queue_inst()+ # Get organization_id for proper HITL queue filtering+ organization_id = UserContext.get_organization_identifier()+ # Get HITL backend queue instance with organization context+ conn = QueueUtils.get_queue_inst(+ {"use_hitl_backend": True, "organization_id": organization_id}+ )++ # Check if this is packet processingif self.packet_id:- # Route to packet queue+ # Route to packet queue (Enterprise feature)+ logger.info(f"Routing {file_name} to packet queue {self.packet_id}")success = conn.enqueue_to_packet(- packet_id=self.packet_id, queue_result=queue_result+ packet_id=self.packet_id,+ queue_result=queue_result,+ organization_id=organization_id,)if not success:error_msg = f"Failed to push {file_name} to packet {self.packet_id}"logger.error(error_msg)raise RuntimeError(error_msg)+ logger.info(+ f"Pushed {file_name} to packet {self.packet_id} with organization_id {organization_id}"+ )return# Route to regular queue@@ -959,9 +970,14 @@ class DestinationConnector(BaseConnector):conn.enqueue_with_ttl(queue_name=q_name, message=queue_result_json, ttl_seconds=ttl_seconds)+ logger.info(+ f"Pushed {file_name} to queue {q_name} with organization_id {organization_id} and TTL {ttl_seconds}"+ )else:conn.enqueue(queue_name=q_name, message=queue_result_json)- logger.info(f"Pushed {file_name} to queue {q_name} with file content")+ logger.info(+ f"Pushed {file_name} to queue {q_name} with organization_id {organization_id}"+ )def _create_queue_result(self,diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.pyindex a3745216d..204518f9a 100644--- a/backend/workflow_manager/endpoint_v2/queue_utils.py+++ b/backend/workflow_manager/endpoint_v2/queue_utils.py@@ -4,6 +4,7 @@ from dataclasses import dataclassfrom enum import Enumfrom typing import Any+from django.conf import settingsfrom utils.constants import Commonfrom workflow_manager.endpoint_v2.exceptions import UnstractQueueException@@ -20,8 +21,47 @@ class QueueResultStatus(Enum):class QueueUtils:+ # HITL connectors registry (lazy-loaded to avoid circular imports)+ _hitl_connectors = {}+@staticmethoddef get_queue_inst(connector_settings: dict[str, Any] = {}) -> UnstractQueue:+ """Get queue connector instance based on configuration.++ For HITL operations, this can return PostgreSQL, Hybrid, or Redis connectors+ based on the HITL_QUEUE_BACKEND setting.+ """+ # Check if caller explicitly wants the default (non-HITL) backend+ # This is used by HybridQueue to get the actual Redis connector without recursion+ force_default = connector_settings.get("force_default_backend", False)+ if force_default:+ # Strip the flag and return default Redis connector+ clean_settings = {+ k: v+ for k, v in connector_settings.items()+ if k != "force_default_backend"+ }+ if not queue_connectors:+ raise UnstractQueueException(detail="Queue connector not exists")+ queue_connector_key = next(iter(queue_connectors))+ connector = queue_connectors[queue_connector_key][Common.METADATA][+ Common.CONNECTOR+ ]+ return connector(clean_settings)++ # Check if this is for HITL operations+ is_hitl = connector_settings.get("use_hitl_backend", False)++ # Auto-detect HITL backend if HITL_QUEUE_BACKEND is set to hybrid/postgresql+ hitl_backend = getattr(settings, "HITL_QUEUE_BACKEND", "redis")+ if not is_hitl and hitl_backend in ("hybrid", "postgresql"):+ is_hitl = True++ if is_hitl:+ # Use HITL-specific queue backend+ return QueueUtils.get_hitl_queue_inst(hitl_backend, connector_settings)++ # Default behavior for non-HITL operationsif not queue_connectors:raise UnstractQueueException(detail="Queue connector not exists")queue_connector_key = next(iter(queue_connectors))@@ -31,6 +71,108 @@ class QueueUtils:connector_class: UnstractQueue = connector(connector_settings)return connector_class+ @staticmethod+ def get_hitl_queue_inst(+ backend: str, connector_settings: dict[str, Any] = {}+ ) -> UnstractQueue:+ """Get HITL-specific queue connector instance with dynamic imports.++ This method uses dynamic imports to avoid hard dependencies on the+ manual_review_v2 pluggable app, allowing graceful degradation when+ the app is not available.++ Args:+ backend: Backend type ('postgresql', 'hybrid', 'redis')+ connector_settings: Connector configuration++ Returns:+ Configured queue connector instance++ Raises:+ UnstractQueueException: When HITL connectors are not available+ """+ # For Redis backend, use default connector+ if backend == "redis":+ # Strip HITL flag to force default (non-HITL) connector path+ non_hitl_settings = dict(connector_settings)+ non_hitl_settings.pop("use_hitl_backend", None)+ return QueueUtils.get_queue_inst(non_hitl_settings)++ # For PostgreSQL and Hybrid backends, try dynamic imports+ try:+ if backend == "postgresql":+ connector_class = QueueUtils._import_hitl_connector("PostgreSQLQueue")+ return connector_class(connector_settings)++ elif backend == "hybrid":+ connector_class = QueueUtils._import_hitl_connector("HybridQueue")+ return connector_class(connector_settings)++ else:+ logger.warning(+ f"Unknown HITL queue backend '{backend}'. "+ f"Valid options: postgresql, hybrid, redis. "+ f"Attempting fallback to hybrid."+ )+ connector_class = QueueUtils._import_hitl_connector("HybridQueue")+ return connector_class(connector_settings)++ except ImportError as e:+ logger.error(+ f"HITL queue backend '{backend}' not available: {e}. "+ f"Make sure 'pluggable_apps.manual_review_v2' is installed and configured."+ )+ raise UnstractQueueException(+ detail=f"HITL queue backend '{backend}' not available. "+ f"Please install the manual_review_v2 app or use 'redis' backend."+ )+ except Exception as e:+ logger.error(f"Failed to initialize HITL queue backend '{backend}': {e}")+ raise UnstractQueueException(+ detail=f"Failed to initialize HITL queue backend '{backend}': {str(e)}"+ )++ @classmethod+ def _import_hitl_connector(cls, connector_name: str):+ """Get HITL connector class from registry (lazy-loaded to avoid circular imports).++ Args:+ connector_name: Name of the connector class to get++ Returns:+ The connector class++ Raises:+ ImportError: When the connector is not available+ """+ # Lazy initialization on first access (avoids circular imports)+ if not cls._hitl_connectors:+ try:+ from pluggable_apps.manual_review_v2.connectors import (+ HybridQueue,+ PostgreSQLQueue,+ )++ cls._hitl_connectors = {+ "PostgreSQLQueue": PostgreSQLQueue,+ "HybridQueue": HybridQueue,+ }+ except ImportError as e:+ logger.debug(f"HITL connectors not available: {e}")++ if not cls._hitl_connectors:+ raise ImportError(+ "HITL connectors not available. Make sure 'pluggable_apps.manual_review_v2' is installed."+ )++ if connector_name not in cls._hitl_connectors:+ available_connectors = list(cls._hitl_connectors.keys())+ raise ImportError(+ f"Unknown HITL connector: {connector_name}. Available: {available_connectors}"+ )++ return cls._hitl_connectors[connector_name]+@staticmethoddef calculate_remaining_ttl(enqueued_at: float, ttl_seconds: int) -> int | None:"""Calculate remaining TTL based on original enqueue time and TTL.diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py b/unstract/filesystem/src/unstract/filesystem/file_storage_config.pyindex a2608ccad..870ed30a0 100644--- a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py@@ -28,9 +28,11 @@ def get_provider(var_name: str, default: str = "minio") -> FileStorageProvider:STORAGE_MAPPING = {FileStorageType.WORKFLOW_EXECUTION: StorageType.SHARED_TEMPORARY,FileStorageType.API_EXECUTION: StorageType.SHARED_TEMPORARY,+ FileStorageType.HITL_FILES: StorageType.SHARED_TEMPORARY, # Use shared temporary for HITL files}FILE_STORAGE_CREDENTIALS_TO_ENV_NAME_MAPPING = {FileStorageType.WORKFLOW_EXECUTION: "WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS",FileStorageType.API_EXECUTION: "API_FILE_STORAGE_CREDENTIALS",+ FileStorageType.HITL_FILES: "HITL_FILES_FILE_STORAGE_CREDENTIALS",}diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py b/unstract/filesystem/src/unstract/filesystem/file_storage_types.pyindex be9889c89..2b6956fe6 100644--- a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_types.py@@ -4,3 +4,4 @@ from enum import Enumclass FileStorageType(Enum):WORKFLOW_EXECUTION = "WORKFLOW_EXECUTION"API_EXECUTION = "API_EXECUTION"+ HITL_FILES = "HITL_FILES"diff --git a/workers/shared/models/result_models.py b/workers/shared/models/result_models.pyindex 48bb932e1..547145b92 100644--- a/workers/shared/models/result_models.py+++ b/workers/shared/models/result_models.py@@ -227,6 +227,7 @@ class QueueResult:ttl_seconds: int | None = Noneextracted_text: str | None = Nonehitl_reason: str | None = None+ hitl_queue_name: str | None = Nonedef __post_init__(self):"""Initialize enqueued_at timestamp if not provided and validate required fields"""@@ -254,5 +255,6 @@ class QueueResult:"ttl_seconds": self.ttl_seconds,"extracted_text": self.extracted_text,"hitl_reason": self.hitl_reason,+ "hitl_queue_name": self.hitl_queue_name,}return result_dictdiff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.pyindex 97851c9b8..9207556db 100644--- a/workers/shared/workflow/destination_connector.py+++ b/workers/shared/workflow/destination_connector.py@@ -1415,8 +1415,16 @@ class WorkerDestinationConnector:return Nonefile_type = file_storage.mime_type(path=output_file_path)+ # Note: INFILE has no extension, so mime_type may return text/plain+ # Allow text/plain for both JSON and TXT since we validate content anyway+ allowed_mime_types = [+ EXT_MIME_MAP[ToolOutputType.JSON.lower()],+ EXT_MIME_MAP[ToolOutputType.TXT.lower()],+ "text/plain", # INFILE has no extension+ ]+if output_type == ToolOutputType.JSON:- if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:+ if file_type not in allowed_mime_types:msg = f"Expected tool output type: JSON, got: '{file_type}'"logger.error(msg)raise Exception(msg)@@ -1424,7 +1432,7 @@ class WorkerDestinationConnector:result = json.loads(file_content)return resultelif output_type == ToolOutputType.TXT:- if file_type != EXT_MIME_MAP[ToolOutputType.TXT.lower()]:+ if file_type not in allowed_mime_types:msg = f"Expected tool output type: TXT, got: '{file_type}'"logger.error(msg)raise Exception(msg)@@ -1799,6 +1807,7 @@ class WorkerDestinationConnector:extracted_text=extracted_text,ttl_seconds=ttl_seconds,hitl_reason=hitl_reason,+ hitl_queue_name=self.hitl_queue_name,)# Only include file_content if provided (backend API will handle it)diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.pyindex c6be2830e..b19c6c2ea 100644--- a/workers/shared/workflow/execution/service.py+++ b/workers/shared/workflow/execution/service.py@@ -1424,6 +1424,7 @@ class WorkerWorkflowExecutionService:)processing_error = str(dest_error)output_result = None+ metadata = None # Ensure metadata is defined for file history creation# Handle metadata for API workflows (matching backend pattern)execution_metadata = None