Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2663 [FEAT] moving away from redis hitl file handling improvements (#1550)

vishnuszipstack38b3ce1fc2ca
diff --git a/backend/sample.env b/backend/sample.env
index 4001f114c..5413c2c5c 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -231,5 +231,8 @@ RUNNER_POLLING_INTERVAL_SECONDS=2
# Examples: 900 (15 min), 1800 (30 min), 3600 (60 min)
MIN_SCHEDULE_INTERVAL_SECONDS=1800
+# HITL Files Storage Configuration
+HITL_FILES_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endpoint_url": "http://unstract-minio:9000", "key": "minio", "secret": "minio123"}}'
+
# File active cache redis db
FILE_ACTIVE_CACHE_REDIS_DB=0
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index 44e2ac853..f86f76827 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -290,7 +290,6 @@ class DestinationConnector(BaseConnector):
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
tool_execution_result: str | None = None
-
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
@@ -940,18 +939,30 @@ class DestinationConnector(BaseConnector):
q_name: Queue name for regular queue
ttl_seconds: TTL in seconds (optional, for regular queue)
"""
- # Get queue instance
- conn = QueueUtils.get_queue_inst()
+ # Get organization_id for proper HITL queue filtering
+ organization_id = UserContext.get_organization_identifier()
+ # Get HITL backend queue instance with organization context
+ conn = QueueUtils.get_queue_inst(
+ {"use_hitl_backend": True, "organization_id": organization_id}
+ )
+
+ # Check if this is packet processing
if self.packet_id:
- # Route to packet queue
+ # Route to packet queue (Enterprise feature)
+ logger.info(f"Routing {file_name} to packet queue {self.packet_id}")
success = conn.enqueue_to_packet(
- packet_id=self.packet_id, queue_result=queue_result
+ packet_id=self.packet_id,
+ queue_result=queue_result,
+ organization_id=organization_id,
)
if not success:
error_msg = f"Failed to push {file_name} to packet {self.packet_id}"
logger.error(error_msg)
raise RuntimeError(error_msg)
+ logger.info(
+ f"Pushed {file_name} to packet {self.packet_id} with organization_id {organization_id}"
+ )
return
# Route to regular queue
@@ -959,9 +970,14 @@ class DestinationConnector(BaseConnector):
conn.enqueue_with_ttl(
queue_name=q_name, message=queue_result_json, ttl_seconds=ttl_seconds
)
+ logger.info(
+ f"Pushed {file_name} to queue {q_name} with organization_id {organization_id} and TTL {ttl_seconds}"
+ )
else:
conn.enqueue(queue_name=q_name, message=queue_result_json)
- logger.info(f"Pushed {file_name} to queue {q_name} with file content")
+ logger.info(
+ f"Pushed {file_name} to queue {q_name} with organization_id {organization_id}"
+ )
def _create_queue_result(
self,
diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.py
index a3745216d..204518f9a 100644
--- a/backend/workflow_manager/endpoint_v2/queue_utils.py
+++ b/backend/workflow_manager/endpoint_v2/queue_utils.py
@@ -4,6 +4,7 @@ from dataclasses import dataclass
from enum import Enum
from typing import Any
+from django.conf import settings
from utils.constants import Common
from workflow_manager.endpoint_v2.exceptions import UnstractQueueException
@@ -20,8 +21,47 @@ class QueueResultStatus(Enum):
class QueueUtils:
+ # HITL connectors registry (lazy-loaded to avoid circular imports)
+ _hitl_connectors = {}
+
@staticmethod
def get_queue_inst(connector_settings: dict[str, Any] = {}) -> UnstractQueue:
+ """Get queue connector instance based on configuration.
+
+ For HITL operations, this can return PostgreSQL, Hybrid, or Redis connectors
+ based on the HITL_QUEUE_BACKEND setting.
+ """
+ # Check if caller explicitly wants the default (non-HITL) backend
+ # This is used by HybridQueue to get the actual Redis connector without recursion
+ force_default = connector_settings.get("force_default_backend", False)
+ if force_default:
+ # Strip the flag and return default Redis connector
+ clean_settings = {
+ k: v
+ for k, v in connector_settings.items()
+ if k != "force_default_backend"
+ }
+ if not queue_connectors:
+ raise UnstractQueueException(detail="Queue connector not exists")
+ queue_connector_key = next(iter(queue_connectors))
+ connector = queue_connectors[queue_connector_key][Common.METADATA][
+ Common.CONNECTOR
+ ]
+ return connector(clean_settings)
+
+ # Check if this is for HITL operations
+ is_hitl = connector_settings.get("use_hitl_backend", False)
+
+ # Auto-detect HITL backend if HITL_QUEUE_BACKEND is set to hybrid/postgresql
+ hitl_backend = getattr(settings, "HITL_QUEUE_BACKEND", "redis")
+ if not is_hitl and hitl_backend in ("hybrid", "postgresql"):
+ is_hitl = True
+
+ if is_hitl:
+ # Use HITL-specific queue backend
+ return QueueUtils.get_hitl_queue_inst(hitl_backend, connector_settings)
+
+ # Default behavior for non-HITL operations
if not queue_connectors:
raise UnstractQueueException(detail="Queue connector not exists")
queue_connector_key = next(iter(queue_connectors))
@@ -31,6 +71,108 @@ class QueueUtils:
connector_class: UnstractQueue = connector(connector_settings)
return connector_class
+ @staticmethod
+ def get_hitl_queue_inst(
+ backend: str, connector_settings: dict[str, Any] = {}
+ ) -> UnstractQueue:
+ """Get HITL-specific queue connector instance with dynamic imports.
+
+ This method uses dynamic imports to avoid hard dependencies on the
+ manual_review_v2 pluggable app, allowing graceful degradation when
+ the app is not available.
+
+ Args:
+ backend: Backend type ('postgresql', 'hybrid', 'redis')
+ connector_settings: Connector configuration
+
+ Returns:
+ Configured queue connector instance
+
+ Raises:
+ UnstractQueueException: When HITL connectors are not available
+ """
+ # For Redis backend, use default connector
+ if backend == "redis":
+ # Strip HITL flag to force default (non-HITL) connector path
+ non_hitl_settings = dict(connector_settings)
+ non_hitl_settings.pop("use_hitl_backend", None)
+ return QueueUtils.get_queue_inst(non_hitl_settings)
+
+ # For PostgreSQL and Hybrid backends, try dynamic imports
+ try:
+ if backend == "postgresql":
+ connector_class = QueueUtils._import_hitl_connector("PostgreSQLQueue")
+ return connector_class(connector_settings)
+
+ elif backend == "hybrid":
+ connector_class = QueueUtils._import_hitl_connector("HybridQueue")
+ return connector_class(connector_settings)
+
+ else:
+ logger.warning(
+ f"Unknown HITL queue backend '{backend}'. "
+ f"Valid options: postgresql, hybrid, redis. "
+ f"Attempting fallback to hybrid."
+ )
+ connector_class = QueueUtils._import_hitl_connector("HybridQueue")
+ return connector_class(connector_settings)
+
+ except ImportError as e:
+ logger.error(
+ f"HITL queue backend '{backend}' not available: {e}. "
+ f"Make sure 'pluggable_apps.manual_review_v2' is installed and configured."
+ )
+ raise UnstractQueueException(
+ detail=f"HITL queue backend '{backend}' not available. "
+ f"Please install the manual_review_v2 app or use 'redis' backend."
+ )
+ except Exception as e:
+ logger.error(f"Failed to initialize HITL queue backend '{backend}': {e}")
+ raise UnstractQueueException(
+ detail=f"Failed to initialize HITL queue backend '{backend}': {str(e)}"
+ )
+
+ @classmethod
+ def _import_hitl_connector(cls, connector_name: str):
+ """Get HITL connector class from registry (lazy-loaded to avoid circular imports).
+
+ Args:
+ connector_name: Name of the connector class to get
+
+ Returns:
+ The connector class
+
+ Raises:
+ ImportError: When the connector is not available
+ """
+ # Lazy initialization on first access (avoids circular imports)
+ if not cls._hitl_connectors:
+ try:
+ from pluggable_apps.manual_review_v2.connectors import (
+ HybridQueue,
+ PostgreSQLQueue,
+ )
+
+ cls._hitl_connectors = {
+ "PostgreSQLQueue": PostgreSQLQueue,
+ "HybridQueue": HybridQueue,
+ }
+ except ImportError as e:
+ logger.debug(f"HITL connectors not available: {e}")
+
+ if not cls._hitl_connectors:
+ raise ImportError(
+ "HITL connectors not available. Make sure 'pluggable_apps.manual_review_v2' is installed."
+ )
+
+ if connector_name not in cls._hitl_connectors:
+ available_connectors = list(cls._hitl_connectors.keys())
+ raise ImportError(
+ f"Unknown HITL connector: {connector_name}. Available: {available_connectors}"
+ )
+
+ return cls._hitl_connectors[connector_name]
+
@staticmethod
def calculate_remaining_ttl(enqueued_at: float, ttl_seconds: int) -> int | None:
"""Calculate remaining TTL based on original enqueue time and TTL.
diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py
index a2608ccad..870ed30a0 100644
--- a/unstract/filesystem/src/unstract/filesystem/file_storage_config.py
+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_config.py
@@ -28,9 +28,11 @@ def get_provider(var_name: str, default: str = "minio") -> FileStorageProvider:
STORAGE_MAPPING = {
FileStorageType.WORKFLOW_EXECUTION: StorageType.SHARED_TEMPORARY,
FileStorageType.API_EXECUTION: StorageType.SHARED_TEMPORARY,
+ FileStorageType.HITL_FILES: StorageType.SHARED_TEMPORARY, # Use shared temporary for HITL files
}
FILE_STORAGE_CREDENTIALS_TO_ENV_NAME_MAPPING = {
FileStorageType.WORKFLOW_EXECUTION: "WORKFLOW_EXECUTION_FILE_STORAGE_CREDENTIALS",
FileStorageType.API_EXECUTION: "API_FILE_STORAGE_CREDENTIALS",
+ FileStorageType.HITL_FILES: "HITL_FILES_FILE_STORAGE_CREDENTIALS",
}
diff --git a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py b/unstract/filesystem/src/unstract/filesystem/file_storage_types.py
index be9889c89..2b6956fe6 100644
--- a/unstract/filesystem/src/unstract/filesystem/file_storage_types.py
+++ b/unstract/filesystem/src/unstract/filesystem/file_storage_types.py
@@ -4,3 +4,4 @@ from enum import Enum
class FileStorageType(Enum):
WORKFLOW_EXECUTION = "WORKFLOW_EXECUTION"
API_EXECUTION = "API_EXECUTION"
+ HITL_FILES = "HITL_FILES"
diff --git a/workers/shared/models/result_models.py b/workers/shared/models/result_models.py
index 48bb932e1..547145b92 100644
--- a/workers/shared/models/result_models.py
+++ b/workers/shared/models/result_models.py
@@ -227,6 +227,7 @@ class QueueResult:
ttl_seconds: int | None = None
extracted_text: str | None = None
hitl_reason: str | None = None
+ hitl_queue_name: str | None = None
def __post_init__(self):
"""Initialize enqueued_at timestamp if not provided and validate required fields"""
@@ -254,5 +255,6 @@ class QueueResult:
"ttl_seconds": self.ttl_seconds,
"extracted_text": self.extracted_text,
"hitl_reason": self.hitl_reason,
+ "hitl_queue_name": self.hitl_queue_name,
}
return result_dict
diff --git a/workers/shared/workflow/destination_connector.py b/workers/shared/workflow/destination_connector.py
index 97851c9b8..9207556db 100644
--- a/workers/shared/workflow/destination_connector.py
+++ b/workers/shared/workflow/destination_connector.py
@@ -1415,8 +1415,16 @@ class WorkerDestinationConnector:
return None
file_type = file_storage.mime_type(path=output_file_path)
+ # Note: INFILE has no extension, so mime_type may return text/plain
+ # Allow text/plain for both JSON and TXT since we validate content anyway
+ allowed_mime_types = [
+ EXT_MIME_MAP[ToolOutputType.JSON.lower()],
+ EXT_MIME_MAP[ToolOutputType.TXT.lower()],
+ "text/plain", # INFILE has no extension
+ ]
+
if output_type == ToolOutputType.JSON:
- if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
+ if file_type not in allowed_mime_types:
msg = f"Expected tool output type: JSON, got: '{file_type}'"
logger.error(msg)
raise Exception(msg)
@@ -1424,7 +1432,7 @@ class WorkerDestinationConnector:
result = json.loads(file_content)
return result
elif output_type == ToolOutputType.TXT:
- if file_type != EXT_MIME_MAP[ToolOutputType.TXT.lower()]:
+ if file_type not in allowed_mime_types:
msg = f"Expected tool output type: TXT, got: '{file_type}'"
logger.error(msg)
raise Exception(msg)
@@ -1799,6 +1807,7 @@ class WorkerDestinationConnector:
extracted_text=extracted_text,
ttl_seconds=ttl_seconds,
hitl_reason=hitl_reason,
+ hitl_queue_name=self.hitl_queue_name,
)
# Only include file_content if provided (backend API will handle it)
diff --git a/workers/shared/workflow/execution/service.py b/workers/shared/workflow/execution/service.py
index c6be2830e..b19c6c2ea 100644
--- a/workers/shared/workflow/execution/service.py
+++ b/workers/shared/workflow/execution/service.py
@@ -1424,6 +1424,7 @@ class WorkerWorkflowExecutionService:
)
processing_error = str(dest_error)
output_result = None
+ metadata = None # Ensure metadata is defined for file history creation
# Handle metadata for API workflows (matching backend pattern)
execution_metadata = None