LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import ast
import base64
import json
import logging
import os
from typing import Any
from connector_v2.models import ConnectorInstance
from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil
from rest_framework.exceptions import APIException
from usage_v2.helper import UsageHelper
from utils.user_context import UserContext
from workflow_manager.endpoint_v2.base_connector import BaseConnector
from workflow_manager.endpoint_v2.constants import (
ApiDeploymentResultStatus,
DestinationKey,
)
from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
from workflow_manager.endpoint_v2.dto import DestinationConfig, FileHash
from workflow_manager.endpoint_v2.exceptions import (
DestinationConnectorNotConfigured,
InvalidDestinationConnectionType,
InvalidToolOutputType,
MissingDestinationConnectionType,
ToolOutputTypeMismatch,
)
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.queue_utils import (
QueueResult,
QueueResultStatus,
QueueUtils,
)
from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow
from backend.exceptions import UnstractFSException
from unstract.connectors.exceptions import ConnectorError
from unstract.filesystem import FileStorageType, FileSystem
from unstract.sdk1.constants import ToolExecKey
from unstract.sdk1.tool.mime_types import EXT_MIME_MAP
from unstract.workflow_execution.constants import ToolOutputType
logger = logging.getLogger(__name__)
class DestinationConnector(BaseConnector):
"""A class representing a Destination connector for a workflow.
This class extends the BaseConnector class and provides methods for
interacting with different types of destination connectors,
such as file system connectors and API connectors and DB connectors.
Attributes:
workflow (Workflow): The workflow associated with
the destination connector.
"""
def __init__(
self,
workflow: Workflow,
execution_id: str,
workflow_log: WorkflowLog,
use_file_history: bool,
file_execution_id: str | None = None,
hitl_queue_name: str | None = None,
packet_id: str | None = None,
) -> None:
"""Initialize a DestinationConnector object.
Args:
workflow (Workflow): _description_
"""
organization_id = UserContext.get_organization_identifier()
super().__init__(workflow.id, execution_id, organization_id, file_execution_id)
self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
self.source_endpoint = self._get_source_endpoint_for_workflow(workflow=workflow)
self.execution_id = execution_id
self.queue_results: list[dict[str, Any]] = []
self.is_api: bool = (
self.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
)
self.workflow_log = workflow_log
self.use_file_history = use_file_history
self.hitl_queue_name = hitl_queue_name
self.packet_id = packet_id
self.workflow = workflow
def _get_endpoint_for_workflow(
self,
workflow: Workflow,
) -> WorkflowEndpoint:
"""Get WorkflowEndpoint instance.
Args:
workflow (Workflow): Workflow associated with the
destination connector.
Returns:
WorkflowEndpoint: WorkflowEndpoint instance.
"""
endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
)
return endpoint
def _get_source_endpoint_for_workflow(
self,
workflow: Workflow,
) -> WorkflowEndpoint:
"""Get WorkflowEndpoint instance.
Args:
workflow (Workflow): Workflow associated with the
destination connector.
Returns:
WorkflowEndpoint: WorkflowEndpoint instance.
"""
endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
)
return endpoint
def validate(self) -> None:
connection_type = self.endpoint.connection_type
connector: ConnectorInstance | None = self.endpoint.connector_instance
if connection_type is None:
error_msg = "Missing destination connection type"
self.workflow_log.log_error(logger, error_msg)
raise MissingDestinationConnectionType()
if connection_type not in WorkflowEndpoint.ConnectionType.values:
error_msg = f"Invalid destination connection type: {connection_type}"
self.workflow_log.log_error(logger, error_msg)
raise InvalidDestinationConnectionType()
# Check if connector is required but missing
requires_connector = (
connection_type != WorkflowEndpoint.ConnectionType.API
and connection_type != WorkflowEndpoint.ConnectionType.MANUALREVIEW
)
if requires_connector and connector is None:
error_msg = "Destination connector not configured"
self.workflow_log.log_error(logger, error_msg)
raise DestinationConnectorNotConfigured()
# Validate database connection if it's a database destination
if connection_type == WorkflowEndpoint.ConnectionType.DATABASE and connector:
try:
# Get database class and test connection
db_class = DatabaseUtils.get_db_class(
connector_id=connector.connector_id,
connector_settings=connector.connector_metadata,
)
engine = db_class.get_engine()
self.workflow_log.log_info(logger, "Database connection test successful")
if hasattr(engine, "close"):
engine.close()
except ConnectorError as e:
error_msg = f"Database connector validation failed: {e}"
self.workflow_log.log_error(logger, error_msg)
logger.exception(error_msg)
raise
except Exception as e:
error_msg = f"Unexpected error during database validation: {e}"
self.workflow_log.log_error(logger, error_msg)
logger.exception(error_msg)
raise
def _should_handle_hitl(
self,
file_name: str,
file_hash: FileHash,
workflow: Workflow,
input_file_path: str,
file_execution_id: str,
) -> bool:
"""Determines if HITL processing should be performed, returning True if data was pushed to the queue.
The logic is:
1. packet_id takes precedence - always push to packet queue
2. hitl_queue_name with API rules:
- If no API rules configured → Push ALL to HITL (backward compatible)
- If API rules exist → Evaluate rules, push only if rules match
3. Database destination → Evaluate DB rules
"""
# Check packet_id first - it takes precedence over hitl_queue_name
if self.packet_id:
logger.info(
f"API packet override: pushing to packet queue for file {file_name}"
)
self._push_data_to_queue(
file_name=file_name,
workflow=workflow,
input_file_path=input_file_path,
file_execution_id=file_execution_id,
)
return True
# Check if API deployment requested HITL override with hitl_queue_name
if self.hitl_queue_name:
# Check if API rules are configured for this workflow
if WorkflowUtil.has_api_rules(workflow):
# API rules exist - evaluate them
execution_result = self.get_tool_execution_result()
if WorkflowUtil.validate_rule_engine(
execution_result,
workflow,
file_hash.file_destination,
rule_type="API",
):
logger.info(
f"API rules matched: pushing to queue for file {file_name}"
)
self._push_data_to_queue(
file_name=file_name,
workflow=workflow,
input_file_path=input_file_path,
file_execution_id=file_execution_id,
)
return True
else:
logger.info(
f"API rules not matched: returning result for file {file_name}"
)
return False
else:
# No API rules configured - backward compatible behavior: push ALL to HITL
logger.info(
f"No API rules configured: pushing to queue for file {file_name}"
)
self._push_data_to_queue(
file_name=file_name,
workflow=workflow,
input_file_path=input_file_path,
file_execution_id=file_execution_id,
)
return True
# Skip HITL validation if we're using file_history and no execution result is available
if self.is_api and self.use_file_history:
return False
# Otherwise use existing workflow-based HITL logic for DB destinations
execution_result = self.get_tool_execution_result()
if WorkflowUtil.validate_rule_engine(
execution_result, workflow, file_hash.file_destination, rule_type="DB"
):
self._push_data_to_queue(
file_name=file_name,
workflow=workflow,
input_file_path=input_file_path,
file_execution_id=file_execution_id,
)
return True
return False
def _push_data_to_queue(
self,
file_name: str,
workflow: Workflow,
input_file_path: str,
file_execution_id: str,
) -> None:
result = self.get_tool_execution_result()
meta_data = self.get_metadata()
self._push_to_queue(
file_name=file_name,
workflow=workflow,
result=result,
input_file_path=input_file_path,
meta_data=meta_data,
file_execution_id=file_execution_id,
)
def handle_output(
self,
file_name: str,
file_hash: FileHash,
file_history: FileHistory | None,
workflow: Workflow,
input_file_path: str,
file_execution_id: str = None,
error: str | None = None,
) -> str | None:
"""Handle the output based on the connection type."""
connection_type = self.endpoint.connection_type
tool_execution_result: str | None = None
if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
self.copy_output_to_output_directory()
elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
# For error cases, skip HITL and directly insert error record
if error:
self.insert_into_db(input_file_path=input_file_path, error=error)
else:
# For success cases, check HITL first, then insert if not HITL
if not self._should_handle_hitl(
file_name=file_name,
file_hash=file_hash,
workflow=workflow,
input_file_path=input_file_path,
file_execution_id=file_execution_id,
):
self.insert_into_db(input_file_path=input_file_path, error=error)
elif connection_type == WorkflowEndpoint.ConnectionType.API:
logger.info(f"API connection type detected for file {file_name}")
# Check for HITL (Manual Review Queue) override for API deployments
if not self._should_handle_hitl(
file_name=file_name,
file_hash=file_hash,
workflow=workflow,
input_file_path=input_file_path,
file_execution_id=file_execution_id,
):
logger.info(
f"No HITL override, getting tool execution result for {file_name}"
)
tool_execution_result = self.get_tool_execution_result(file_history)
elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
self._push_data_to_queue(
file_name,
workflow,
input_file_path,
file_execution_id,
)
self.workflow_log.publish_log(
message=f"File '{file_name}' processed successfully"
)
return tool_execution_result
def copy_output_to_output_directory(self) -> None:
"""Copy output to the destination directory."""
connector: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector.connector_metadata
destination_configurations: dict[str, Any] = self.endpoint.configuration
root_path = str(connector_settings.get(DestinationKey.PATH, ""))
output_directory = str(
destination_configurations.get(DestinationKey.OUTPUT_FOLDER, "/")
)
destination_fs = self.get_fs_connector(
settings=connector_settings, connector_id=connector.connector_id
)
output_directory = destination_fs.get_connector_root_dir(
input_dir=output_directory, root_path=root_path
)
logger.debug(f"destination output directory {output_directory}")
destination_volume_path = os.path.join(
self.file_execution_dir, ToolExecKey.OUTPUT_DIR
)
try:
destination_fs.create_dir_if_not_exists(input_dir=output_directory)
# Traverse local directory and create the same structure in the
# output_directory
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
fs = file_system.get_file_storage()
dir_path = fs.walk(str(destination_volume_path))
for root, dirs, files in dir_path:
for dir_name in dirs:
current_dir = os.path.join(
output_directory,
os.path.relpath(root, destination_volume_path),
dir_name,
)
destination_fs.create_dir_if_not_exists(input_dir=current_dir)
for file_name in files:
source_path = os.path.join(root, file_name)
destination_path = os.path.join(
output_directory,
os.path.relpath(root, destination_volume_path),
file_name,
)
destination_fs.upload_file_to_storage(
source_path=source_path, destination_path=destination_path
)
except ConnectorError as e:
raise UnstractFSException(core_err=e) from e
def insert_into_db(self, input_file_path: str, error: str | None = None) -> None:
"""Insert data into the database."""
connector_instance: ConnectorInstance = self.endpoint.connector_instance
connector_settings: dict[str, Any] = connector_instance.connector_metadata
destination_configurations: dict[str, Any] = self.endpoint.configuration
table_name: str = str(destination_configurations.get(DestinationKey.TABLE))
include_agent: bool = bool(
destination_configurations.get(DestinationKey.INCLUDE_AGENT, False)
)
include_timestamp = bool(
destination_configurations.get(DestinationKey.INCLUDE_TIMESTAMP, False)
)
agent_name = str(destination_configurations.get(DestinationKey.AGENT_NAME))
column_mode = str(destination_configurations.get(DestinationKey.COLUMN_MODE))
single_column_name = str(
destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data")
)
file_path_name = str(
destination_configurations.get(DestinationKey.FILE_PATH, "file_path")
)
execution_id_name = str(
destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
)
data = self.get_tool_execution_result() if not error else None
metadata = self.get_combined_metadata()
if not data and not error:
logger.info("No data obtained from tool to insert into destination DB.")
return
# Log when we're proceeding with error insertion
if error and not data:
logger.info(
f"Proceeding with error record insertion for {input_file_path}: {error}"
)
if isinstance(data, dict):
data.pop("metadata", None)
db_class = DatabaseUtils.get_db_class(
connector_id=connector_instance.connector_id,
connector_settings=connector_settings,
)
engine = db_class.get_engine()
table_info = db_class.get_information_schema(table_name=table_name)
logger.info(
f"destination connector table_name: {table_name} with table_info: {table_info}"
)
if table_info:
if db_class.has_no_metadata(table_info=table_info):
table_info = DatabaseUtils.migrate_table_to_v2(
db_class=db_class,
engine=engine,
table_name=table_name,
column_name=single_column_name,
)
values = DatabaseUtils.get_columns_and_values(
column_mode_str=column_mode,
data=data,
metadata=metadata,
include_timestamp=include_timestamp,
include_agent=include_agent,
agent_name=agent_name,
single_column_name=single_column_name,
file_path_name=file_path_name,
execution_id_name=execution_id_name,
table_info=table_info,
file_path=input_file_path,
execution_id=self.execution_id,
error=error,
)
logger.debug(f"destination.py values: {values}")
try:
# Reuse the same db_class and engine created earlier
DatabaseUtils.create_table_if_not_exists(
db_class=db_class,
engine=engine,
table_name=table_name,
database_entry=values,
)
sql_columns_and_values = DatabaseUtils.get_sql_query_data(
conn_cls=db_class,
table_name=table_name,
values=values,
)
logger.info("destination.py sql_columns_and_values", sql_columns_and_values)
DatabaseUtils.execute_write_query(
db_class=db_class,
engine=engine,
table_name=table_name,
sql_keys=list(sql_columns_and_values.keys()),
sql_values=list(sql_columns_and_values.values()),
)
except ConnectorError as e:
error_msg = f"Database connection failed for {input_file_path}: {str(e)}"
logger.error(error_msg)
raise
except Exception as e:
error_msg = (
f"Failed to insert data into database for {input_file_path}: {str(e)}"
)
logger.error(error_msg)
raise
finally:
self._close_engine(engine, input_file_path)
def _close_engine(self, engine: Any, input_file_path: str) -> None:
"""Safely close database engine."""
if engine:
try:
engine.close()
except Exception as e:
logger.error(
f"Failed to close database engine for {input_file_path}: {str(e)}"
)
def _handle_api_result(
self,
file_name: str,
error: str | None = None,
result: str | None = None,
metadata: dict[str, Any] | None = None,
) -> None:
"""Handle the API result.
This method is responsible for handling the API result.
It appends the file name and result to the 'results' list for API resp.
Args:
file_name (str): The name of the file.
result (Optional[str], optional): The result of the API call.
Defaults to None.
Returns:
None
"""
api_result: dict[str, Any] = {
"file": file_name,
"file_execution_id": self.file_execution_id,
}
if error:
api_result.update(
{"status": ApiDeploymentResultStatus.FAILED, "error": error}
)
else:
if result:
api_result.update(
{
"status": ApiDeploymentResultStatus.SUCCESS,
"result": result,
"metadata": metadata,
}
)
else:
api_result.update(
{"status": ApiDeploymentResultStatus.SUCCESS, "result": ""}
)
self.update_api_results(api_result)
def parse_string(self, original_string: str) -> Any:
"""Parse the given string, attempting to evaluate it as a Python
literal.
ex: a json string to dict method
Parameters:
- original_string (str): The input string to be parsed.
Returns:
- Any: The parsed result. If the string can be evaluated as a Python
literal, the result of the evaluation is returned.
If not, the original string is returned unchanged.
Note:
This function uses `ast.literal_eval` to attempt parsing the string as a
Python literal. If parsing fails due to a SyntaxError or ValueError,
the original string is returned.
Example:
>>> parser.parse_string("42")
42
>>> parser.parse_string("[1, 2, 3]")
[1, 2, 3]
>>> parser.parse_string("Hello, World!")
'Hello, World!'
"""
try:
# Try to evaluate as a Python literal
python_literal = ast.literal_eval(original_string)
return python_literal
except (SyntaxError, ValueError):
# If evaluating as a Python literal fails,
# assume it's a plain string
return original_string
def get_tool_execution_result(
self, file_history: FileHistory | None = None
) -> Any | None:
"""Get result data from the output file.
Returns:
Union[dict[str, Any], str]: Result data.
"""
return self.get_tool_execution_result_from_metadata(file_history=file_history)
def get_tool_execution_result_from_metadata(
self, file_history: FileHistory | None = None
) -> Any | None:
"""Get result data from the output file.
Returns:
Union[dict[str, Any], str]: Result data.
"""
if file_history and file_history.result:
return self.parse_string(file_history.result)
output_file = self.infile
metadata: dict[str, Any] = self.get_workflow_metadata()
output_type = self.get_output_type(metadata)
result: dict[str, Any] | str = ""
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()
try:
# TODO: SDK handles validation; consider removing here.
file_type = file_storage.mime_type(path=output_file)
if output_type == ToolOutputType.JSON:
if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
msg = f"Expected tool output type: JSON, got: '{file_type}'"
logger.error(msg)
raise ToolOutputTypeMismatch(detail=msg)
file_content = file_storage.read(output_file, mode="r")
result = json.loads(file_content)
elif output_type == ToolOutputType.TXT:
if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
msg = f"Expected tool output type: TXT, got: '{file_type}'"
logger.error(msg)
raise ToolOutputTypeMismatch(detail=msg)
file_content = file_storage.read(output_file, mode="r")
result = file_content.encode("utf-8").decode("unicode-escape")
else:
raise InvalidToolOutputType()
except (FileNotFoundError, json.JSONDecodeError) as err:
msg = f"Error while getting result from the tool: {err}"
logger.error(msg)
raise APIException(detail=msg)
return result
def has_valid_metadata(self, metadata: Any) -> bool:
# Check if metadata is not None and is either a non-empty dict or valid string
if metadata is None:
return False
# Handle dict metadata (which is valid and contains extracted_text)
if isinstance(metadata, dict):
return bool(metadata) # Return True if dict is not empty
# Handle string metadata
if isinstance(metadata, str):
if metadata.strip().lower() == "none" or not metadata.strip():
return False
return True
# For other types, consider them valid if they're truthy
return bool(metadata)
def get_metadata(
self, file_history: FileHistory | None = None
) -> dict[str, Any] | None:
"""Get metadata from the output file.
Returns:
Union[dict[str, Any], str]: Metadata.
"""
if file_history:
if self.has_valid_metadata(file_history.metadata):
return self.parse_string(file_history.metadata)
else:
return None
metadata: dict[str, Any] = self.get_workflow_metadata()
return metadata
def get_combined_metadata(self) -> dict[str, Any]:
"""Get combined workflow and usage metadata.
Returns:
dict[str, Any]: Combined metadata including workflow and usage data.
"""
# Get workflow metadata
workflow_metadata = self.get_metadata()
# Get file_execution_id from metadata
file_execution_id = workflow_metadata.get("file_execution_id")
if not file_execution_id:
return workflow_metadata
usage_metadata = UsageHelper.get_aggregated_token_count(file_execution_id)
# Combine both metadata
workflow_metadata["usage"] = usage_metadata
workflow_metadata["total_pages_processed"] = (
UsageHelper.get_aggregated_pages_processed(run_id=file_execution_id)
)
return workflow_metadata
def delete_file_execution_directory(self) -> None:
"""Delete the file execution directory.
Returns:
None
"""
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()
if self.file_execution_dir and file_storage.exists(self.file_execution_dir):
file_storage.rm(self.file_execution_dir, recursive=True)
@classmethod
def delete_execution_and_api_storage_dir(
cls, workflow_id: str, execution_id: str
) -> None:
"""Delete the execution and api storage directories.
Returns:
None
"""
cls.delete_execution_directory(workflow_id, execution_id)
cls.delete_api_storage_dir(workflow_id, execution_id)
@classmethod
def delete_api_storage_dir(cls, workflow_id: str, execution_id: str) -> None:
"""Delete the api storage path.
Returns:
None
"""
api_storage_dir = cls.get_api_storage_dir_path(
workflow_id=workflow_id, execution_id=execution_id
)
file_system = FileSystem(FileStorageType.API_EXECUTION)
file_storage = file_system.get_file_storage()
if file_storage.exists(api_storage_dir):
file_storage.rm(api_storage_dir, recursive=True)
logger.info(f"API storage directory deleted: {api_storage_dir}")
@classmethod
def delete_execution_directory(cls, workflow_id: str, execution_id: str) -> None:
"""Delete the execution directory.
Returns:
None
"""
execution_dir = cls.get_execution_dir_path(
workflow_id=workflow_id, execution_id=execution_id
)
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()
if file_storage.exists(execution_dir):
file_storage.rm(execution_dir, recursive=True)
logger.info(f"Execution directory deleted: {execution_dir}")
@classmethod
def create_endpoint_for_workflow(
cls,
workflow: Workflow,
) -> None:
"""Create a workflow endpoint for the destination.
Args:
workflow (Workflow): Workflow for which the endpoint is created.
"""
endpoint = WorkflowEndpoint(
workflow=workflow,
endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
)
endpoint.save()
@classmethod
def get_json_schema_for_database(cls) -> dict[str, Any]:
"""Get JSON schema for the database.
Returns:
dict[str, Any]: JSON schema for the database.
"""
schema_path = os.path.join(os.path.dirname(__file__), "static", "dest", "db.json")
return cls.get_json_schema(file_path=schema_path)
@classmethod
def get_json_schema_for_file_system(cls) -> dict[str, Any]:
"""Get JSON schema for the file system.
Returns:
dict[str, Any]: JSON schema for the file system.
"""
schema_path = os.path.join(
os.path.dirname(__file__), "static", "dest", "file.json"
)
return cls.get_json_schema(file_path=schema_path)
@classmethod
def get_json_schema_for_api(cls) -> dict[str, Any]:
"""Json schema for api.
Returns:
dict[str, Any]: _description_
"""
schema_path = os.path.join(
os.path.dirname(__file__), "static", "dest", "api.json"
)
return cls.get_json_schema(file_path=schema_path)
def get_config(self) -> DestinationConfig:
"""Get serializable configuration for the destination connector.
Returns:
DestinationConfig: Configuration containing all necessary data to reconstruct the connector
"""
return DestinationConfig(
workflow_id=self.workflow.id,
execution_id=self.execution_id,
use_file_history=self.use_file_history,
hitl_queue_name=self.hitl_queue_name,
packet_id=self.packet_id,
)
@classmethod
def from_config(
cls, workflow_log: WorkflowLog, config: DestinationConfig
) -> "DestinationConnector":
"""Create a DestinationConnector instance from configuration.
Args:
config (DestinationConfig): Configuration containing all necessary data to reconstruct the connector
Returns:
DestinationConnector: New instance
"""
logger.info(
f"Creating DestinationConnector from config: hitl_queue_name={config.hitl_queue_name}"
)
# Reconstruct workflow
workflow = Workflow.objects.get(id=config.workflow_id)
# Create destination connector instance
destination = cls(
workflow=workflow,
execution_id=config.execution_id,
workflow_log=workflow_log,
use_file_history=config.use_file_history,
file_execution_id=config.file_execution_id,
hitl_queue_name=config.hitl_queue_name,
packet_id=config.packet_id,
)
return destination
def _get_review_queue_name(self) -> str:
"""Generate review queue name with optional HITL override for manual review processing.
Returns:
str: Queue name in the appropriate format:
- Custom HITL queue: review_queue_{org}_{workflow_id}:{hitl_queue_name}
- Standard queue: review_queue_{org}_{workflow_id}
"""
logger.debug(f"Queue naming - hitl_queue_name={self.hitl_queue_name}")
# Base queue format: review_queue_{org}_{workflow_id}
base_queue_name = f"review_queue_{self.organization_id}_{str(self.workflow_id)}"
if self.hitl_queue_name:
# Custom HITL queue with user-specified name
q_name = f"{base_queue_name}:{self.hitl_queue_name}"
logger.debug(f"Using custom HITL queue: {q_name}")
else:
# Standard queue format for workflow-based processing
q_name = base_queue_name
logger.debug(f"Using standard queue name: {q_name}")
return q_name
def _push_to_queue(
self,
file_name: str,
workflow: Workflow,
result: str | None = None,
input_file_path: str | None = None,
meta_data: dict[str, Any] | None = None,
file_execution_id: str = None,
) -> None:
"""Handle the Manual Review QUEUE result.
This method is responsible for pushing the input file and result to
review queue.
Args:
file_name (str): The name of the file.
workflow (Workflow): The workflow object containing
details about the workflow.
result (Optional[str], optional): The result of the API call.
Defaults to None.
input_file_path (Optional[str], optional):
The path to the input file.
Defaults to None.
meta_data (Optional[dict[str, Any]], optional):
A dictionary containing additional
metadata related to the file. Defaults to None.
Returns:
None
"""
# Handle missing result for packet processing
if not result:
if not self.packet_id:
return
result = json.dumps({"status": "pending", "message": "Awaiting processing"})
# Delegate to appropriate handler based on deployment type
if self.is_api:
self._push_to_queue_for_api_deployment(
file_name, result, input_file_path, file_execution_id, meta_data
)
else:
self._push_to_queue_for_connector(
file_name,
workflow,
result,
input_file_path,
file_execution_id,
meta_data,
)
def _enqueue_to_packet_or_regular_queue(
self,
file_name: str,
queue_result: dict[str, Any],
queue_result_json: str,
q_name: str,
ttl_seconds: int | None = None,
) -> None:
"""Route to packet queue or regular queue based on packet_id.
Args:
file_name: Name of the file being queued
queue_result: Queue result dictionary
queue_result_json: JSON string of queue result
q_name: Queue name for regular queue
ttl_seconds: TTL in seconds (optional, for regular queue)
"""
# Get organization_id for proper HITL queue filtering
organization_id = UserContext.get_organization_identifier()
# Get HITL backend queue instance with organization context
conn = QueueUtils.get_queue_inst(
{"use_hitl_backend": True, "organization_id": organization_id}
)
# Check if this is packet processing
if self.packet_id:
# Route to packet queue (Enterprise feature)
logger.info(f"Routing {file_name} to packet queue {self.packet_id}")
success = conn.enqueue_to_packet(
packet_id=self.packet_id,
queue_result=queue_result,
organization_id=organization_id,
)
if not success:
error_msg = f"Failed to push {file_name} to packet {self.packet_id}"
logger.error(error_msg)
raise RuntimeError(error_msg)
logger.info(
f"Pushed {file_name} to packet {self.packet_id} with organization_id {organization_id}"
)
return
# Route to regular queue
if ttl_seconds:
conn.enqueue_with_ttl(
queue_name=q_name, message=queue_result_json, ttl_seconds=ttl_seconds
)
logger.info(
f"Pushed {file_name} to queue {q_name} with organization_id {organization_id} and TTL {ttl_seconds}"
)
else:
conn.enqueue(queue_name=q_name, message=queue_result_json)
logger.info(
f"Pushed {file_name} to queue {q_name} with organization_id {organization_id}"
)
def _create_queue_result(
self,
file_name: str,
result: str,
file_content_base64: str,
file_execution_id: str,
meta_data: dict[str, Any] | None = None,
ttl_seconds: int | None = None,
hitl_reason: str | None = None,
) -> dict[str, Any]:
"""Create QueueResult dictionary.
Args:
file_name: Name of the file
result: Processing result
file_content_base64: Base64 encoded file content
file_execution_id: File execution ID
meta_data: Optional metadata
ttl_seconds: Optional TTL in seconds
hitl_reason: Optional reason why file was sent to HITL
Returns:
QueueResult as dictionary
"""
whisper_hash = meta_data.get("whisper-hash") if meta_data else None
extracted_text = meta_data.get("extracted_text") if meta_data else None
queue_result_obj = QueueResult(
file=file_name,
status=QueueResultStatus.SUCCESS,
result=result,
workflow_id=str(self.workflow_id),
file_content=file_content_base64,
whisper_hash=whisper_hash,
file_execution_id=file_execution_id,
execution_id=self.execution_id,
extracted_text=extracted_text,
ttl_seconds=ttl_seconds,
hitl_reason=hitl_reason,
)
return queue_result_obj.to_dict()
def _push_to_queue_for_api_deployment(
self,
file_name: str,
result: str,
input_file_path: str,
file_execution_id: str,
meta_data: dict[str, Any] | None = None,
) -> None:
"""Handle queue push for API deployments.
Args:
file_name: Name of the file
result: Processing result
input_file_path: Path to input file
file_execution_id: File execution ID
meta_data: Optional metadata
"""
file_content_base64 = self._read_file_content_for_queue(
input_file_path, file_name
)
q_name = self._get_review_queue_name()
ttl_seconds = WorkflowUtil.get_hitl_ttl_seconds(self.workflow)
queue_result = self._create_queue_result(
file_name=file_name,
result=result,
file_content_base64=file_content_base64,
file_execution_id=file_execution_id,
meta_data=meta_data,
ttl_seconds=ttl_seconds,
)
queue_result_json = json.dumps(queue_result)
self._enqueue_to_packet_or_regular_queue(
file_name, queue_result, queue_result_json, q_name, ttl_seconds
)
def _push_to_queue_for_connector(
self,
file_name: str,
workflow: Workflow,
result: str,
input_file_path: str,
file_execution_id: str,
meta_data: dict[str, Any] | None = None,
) -> None:
"""Handle queue push for connector-based deployments.
Args:
file_name: Name of the file
workflow: Workflow object
result: Processing result
input_file_path: Path to input file
file_execution_id: File execution ID
meta_data: Optional metadata
"""
connector = self.source_endpoint.connector_instance
connector_settings = connector.connector_metadata
source_fs = self.get_fsspec(
settings=connector_settings, connector_id=connector.connector_id
)
with source_fs.open(input_file_path, "rb") as remote_file:
file_content = remote_file.read()
file_content_base64 = base64.b64encode(file_content).decode("utf-8")
q_name = self._get_review_queue_name()
ttl_seconds = WorkflowUtil.get_hitl_ttl_seconds(workflow)
queue_result = self._create_queue_result(
file_name=file_name,
result=result,
file_content_base64=file_content_base64,
file_execution_id=file_execution_id,
meta_data=meta_data,
ttl_seconds=ttl_seconds,
)
queue_result_json = json.dumps(queue_result)
# Validate JSON is not empty
if not queue_result_json or queue_result_json.strip() == "":
logger.error(f"Attempted to enqueue empty JSON with TTL for file {file_name}")
raise ValueError("Cannot enqueue empty JSON message")
self._enqueue_to_packet_or_regular_queue(
file_name, queue_result, queue_result_json, q_name, ttl_seconds
)
def _read_file_content_for_queue(self, input_file_path: str, file_name: str) -> str:
"""Read and encode file content for queue message.
Args:
input_file_path: Path to the file to read
file_name: Name of the file for logging purposes
Returns:
Base64 encoded file content
Raises:
APIException: If file cannot be read or doesn't exist
"""
try:
file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
file_storage = file_system.get_file_storage()
if not file_storage.exists(input_file_path):
raise APIException(f"File not found: {input_file_path}")
file_bytes = file_storage.read(input_file_path, mode="rb")
if isinstance(file_bytes, str):
file_bytes = file_bytes.encode("utf-8")
return base64.b64encode(file_bytes).decode("utf-8")
except Exception as e:
logger.error(f"Failed to read file content for {file_name}: {e}")
raise APIException(f"Failed to read file content for queue: {e}")