Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

42.3 KB
Raw Blame
import ast
import base64
import json
import logging
import os
from typing import Any

from connector_v2.models import ConnectorInstance
from plugins.workflow_manager.workflow_v2.utils import WorkflowUtil
from rest_framework.exceptions import APIException
from usage_v2.helper import UsageHelper
from utils.user_context import UserContext
from workflow_manager.endpoint_v2.base_connector import BaseConnector
from workflow_manager.endpoint_v2.constants import (
    ApiDeploymentResultStatus,
    DestinationKey,
)
from workflow_manager.endpoint_v2.database_utils import DatabaseUtils
from workflow_manager.endpoint_v2.dto import DestinationConfig, FileHash
from workflow_manager.endpoint_v2.exceptions import (
    DestinationConnectorNotConfigured,
    InvalidDestinationConnectionType,
    InvalidToolOutputType,
    MissingDestinationConnectionType,
    ToolOutputTypeMismatch,
)
from workflow_manager.endpoint_v2.models import WorkflowEndpoint
from workflow_manager.endpoint_v2.queue_utils import (
    QueueResult,
    QueueResultStatus,
    QueueUtils,
)
from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.models.file_history import FileHistory
from workflow_manager.workflow_v2.models.workflow import Workflow

from backend.exceptions import UnstractFSException
from unstract.connectors.exceptions import ConnectorError
from unstract.filesystem import FileStorageType, FileSystem
from unstract.sdk1.constants import ToolExecKey
from unstract.sdk1.tool.mime_types import EXT_MIME_MAP
from unstract.workflow_execution.constants import ToolOutputType

logger = logging.getLogger(__name__)


class DestinationConnector(BaseConnector):
    """A class representing a Destination connector for a workflow.

    This class extends the BaseConnector class and provides methods for
    interacting with different types of destination connectors,
    such as file system connectors and API connectors and DB connectors.

    Attributes:
        workflow (Workflow): The workflow associated with
            the destination connector.
    """

    def __init__(
        self,
        workflow: Workflow,
        execution_id: str,
        workflow_log: WorkflowLog,
        use_file_history: bool,
        file_execution_id: str | None = None,
        hitl_queue_name: str | None = None,
        packet_id: str | None = None,
    ) -> None:
        """Initialize a DestinationConnector object.

        Args:
            workflow (Workflow): _description_
        """
        organization_id = UserContext.get_organization_identifier()
        super().__init__(workflow.id, execution_id, organization_id, file_execution_id)
        self.endpoint = self._get_endpoint_for_workflow(workflow=workflow)
        self.source_endpoint = self._get_source_endpoint_for_workflow(workflow=workflow)
        self.execution_id = execution_id
        self.queue_results: list[dict[str, Any]] = []
        self.is_api: bool = (
            self.endpoint.connection_type == WorkflowEndpoint.ConnectionType.API
        )
        self.workflow_log = workflow_log
        self.use_file_history = use_file_history
        self.hitl_queue_name = hitl_queue_name
        self.packet_id = packet_id
        self.workflow = workflow

    def _get_endpoint_for_workflow(
        self,
        workflow: Workflow,
    ) -> WorkflowEndpoint:
        """Get WorkflowEndpoint instance.

        Args:
            workflow (Workflow): Workflow associated with the
                destination connector.

        Returns:
            WorkflowEndpoint: WorkflowEndpoint instance.
        """
        endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
            workflow=workflow,
            endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
        )
        return endpoint

    def _get_source_endpoint_for_workflow(
        self,
        workflow: Workflow,
    ) -> WorkflowEndpoint:
        """Get WorkflowEndpoint instance.

        Args:
            workflow (Workflow): Workflow associated with the
                destination connector.

        Returns:
            WorkflowEndpoint: WorkflowEndpoint instance.
        """
        endpoint: WorkflowEndpoint = WorkflowEndpoint.objects.get(
            workflow=workflow,
            endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,
        )
        return endpoint

    def validate(self) -> None:
        connection_type = self.endpoint.connection_type
        connector: ConnectorInstance | None = self.endpoint.connector_instance

        if connection_type is None:
            error_msg = "Missing destination connection type"
            self.workflow_log.log_error(logger, error_msg)
            raise MissingDestinationConnectionType()
        if connection_type not in WorkflowEndpoint.ConnectionType.values:
            error_msg = f"Invalid destination connection type: {connection_type}"
            self.workflow_log.log_error(logger, error_msg)
            raise InvalidDestinationConnectionType()
        # Check if connector is required but missing
        requires_connector = (
            connection_type != WorkflowEndpoint.ConnectionType.API
            and connection_type != WorkflowEndpoint.ConnectionType.MANUALREVIEW
        )
        if requires_connector and connector is None:
            error_msg = "Destination connector not configured"
            self.workflow_log.log_error(logger, error_msg)
            raise DestinationConnectorNotConfigured()

        # Validate database connection if it's a database destination
        if connection_type == WorkflowEndpoint.ConnectionType.DATABASE and connector:
            try:
                # Get database class and test connection
                db_class = DatabaseUtils.get_db_class(
                    connector_id=connector.connector_id,
                    connector_settings=connector.connector_metadata,
                )
                engine = db_class.get_engine()
                self.workflow_log.log_info(logger, "Database connection test successful")
                if hasattr(engine, "close"):
                    engine.close()
            except ConnectorError as e:
                error_msg = f"Database connector validation failed: {e}"
                self.workflow_log.log_error(logger, error_msg)
                logger.exception(error_msg)
                raise
            except Exception as e:
                error_msg = f"Unexpected error during database validation: {e}"
                self.workflow_log.log_error(logger, error_msg)
                logger.exception(error_msg)
                raise

    def _should_handle_hitl(
        self,
        file_name: str,
        file_hash: FileHash,
        workflow: Workflow,
        input_file_path: str,
        file_execution_id: str,
    ) -> bool:
        """Determines if HITL processing should be performed, returning True if data was pushed to the queue.

        The logic is:
        1. packet_id takes precedence - always push to packet queue
        2. hitl_queue_name with API rules:
           - If no API rules configured → Push ALL to HITL (backward compatible)
           - If API rules exist → Evaluate rules, push only if rules match
        3. Database destination → Evaluate DB rules
        """
        # Check packet_id first - it takes precedence over hitl_queue_name
        if self.packet_id:
            logger.info(
                f"API packet override: pushing to packet queue for file {file_name}"
            )
            self._push_data_to_queue(
                file_name=file_name,
                workflow=workflow,
                input_file_path=input_file_path,
                file_execution_id=file_execution_id,
            )
            return True

        # Check if API deployment requested HITL override with hitl_queue_name
        if self.hitl_queue_name:
            # Check if API rules are configured for this workflow
            if WorkflowUtil.has_api_rules(workflow):
                # API rules exist - evaluate them
                execution_result = self.get_tool_execution_result()
                if WorkflowUtil.validate_rule_engine(
                    execution_result,
                    workflow,
                    file_hash.file_destination,
                    rule_type="API",
                ):
                    logger.info(
                        f"API rules matched: pushing to queue for file {file_name}"
                    )
                    self._push_data_to_queue(
                        file_name=file_name,
                        workflow=workflow,
                        input_file_path=input_file_path,
                        file_execution_id=file_execution_id,
                    )
                    return True
                else:
                    logger.info(
                        f"API rules not matched: returning result for file {file_name}"
                    )
                    return False
            else:
                # No API rules configured - backward compatible behavior: push ALL to HITL
                logger.info(
                    f"No API rules configured: pushing to queue for file {file_name}"
                )
                self._push_data_to_queue(
                    file_name=file_name,
                    workflow=workflow,
                    input_file_path=input_file_path,
                    file_execution_id=file_execution_id,
                )
                return True

        # Skip HITL validation if we're using file_history and no execution result is available
        if self.is_api and self.use_file_history:
            return False

        # Otherwise use existing workflow-based HITL logic for DB destinations
        execution_result = self.get_tool_execution_result()

        if WorkflowUtil.validate_rule_engine(
            execution_result, workflow, file_hash.file_destination, rule_type="DB"
        ):
            self._push_data_to_queue(
                file_name=file_name,
                workflow=workflow,
                input_file_path=input_file_path,
                file_execution_id=file_execution_id,
            )
            return True

        return False

    def _push_data_to_queue(
        self,
        file_name: str,
        workflow: Workflow,
        input_file_path: str,
        file_execution_id: str,
    ) -> None:
        result = self.get_tool_execution_result()
        meta_data = self.get_metadata()
        self._push_to_queue(
            file_name=file_name,
            workflow=workflow,
            result=result,
            input_file_path=input_file_path,
            meta_data=meta_data,
            file_execution_id=file_execution_id,
        )

    def handle_output(
        self,
        file_name: str,
        file_hash: FileHash,
        file_history: FileHistory | None,
        workflow: Workflow,
        input_file_path: str,
        file_execution_id: str = None,
        error: str | None = None,
    ) -> str | None:
        """Handle the output based on the connection type."""
        connection_type = self.endpoint.connection_type
        tool_execution_result: str | None = None
        if connection_type == WorkflowEndpoint.ConnectionType.FILESYSTEM:
            self.copy_output_to_output_directory()

        elif connection_type == WorkflowEndpoint.ConnectionType.DATABASE:
            # For error cases, skip HITL and directly insert error record
            if error:
                self.insert_into_db(input_file_path=input_file_path, error=error)
            else:
                # For success cases, check HITL first, then insert if not HITL
                if not self._should_handle_hitl(
                    file_name=file_name,
                    file_hash=file_hash,
                    workflow=workflow,
                    input_file_path=input_file_path,
                    file_execution_id=file_execution_id,
                ):
                    self.insert_into_db(input_file_path=input_file_path, error=error)

        elif connection_type == WorkflowEndpoint.ConnectionType.API:
            logger.info(f"API connection type detected for file {file_name}")
            # Check for HITL (Manual Review Queue) override for API deployments
            if not self._should_handle_hitl(
                file_name=file_name,
                file_hash=file_hash,
                workflow=workflow,
                input_file_path=input_file_path,
                file_execution_id=file_execution_id,
            ):
                logger.info(
                    f"No HITL override, getting tool execution result for {file_name}"
                )
                tool_execution_result = self.get_tool_execution_result(file_history)

        elif connection_type == WorkflowEndpoint.ConnectionType.MANUALREVIEW:
            self._push_data_to_queue(
                file_name,
                workflow,
                input_file_path,
                file_execution_id,
            )

        self.workflow_log.publish_log(
            message=f"File '{file_name}' processed successfully"
        )
        return tool_execution_result

    def copy_output_to_output_directory(self) -> None:
        """Copy output to the destination directory."""
        connector: ConnectorInstance = self.endpoint.connector_instance
        connector_settings: dict[str, Any] = connector.connector_metadata
        destination_configurations: dict[str, Any] = self.endpoint.configuration
        root_path = str(connector_settings.get(DestinationKey.PATH, ""))

        output_directory = str(
            destination_configurations.get(DestinationKey.OUTPUT_FOLDER, "/")
        )
        destination_fs = self.get_fs_connector(
            settings=connector_settings, connector_id=connector.connector_id
        )
        output_directory = destination_fs.get_connector_root_dir(
            input_dir=output_directory, root_path=root_path
        )
        logger.debug(f"destination output directory {output_directory}")
        destination_volume_path = os.path.join(
            self.file_execution_dir, ToolExecKey.OUTPUT_DIR
        )

        try:
            destination_fs.create_dir_if_not_exists(input_dir=output_directory)
            # Traverse local directory and create the same structure in the
            # output_directory
            file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
            fs = file_system.get_file_storage()
            dir_path = fs.walk(str(destination_volume_path))

            for root, dirs, files in dir_path:
                for dir_name in dirs:
                    current_dir = os.path.join(
                        output_directory,
                        os.path.relpath(root, destination_volume_path),
                        dir_name,
                    )
                    destination_fs.create_dir_if_not_exists(input_dir=current_dir)

                for file_name in files:
                    source_path = os.path.join(root, file_name)
                    destination_path = os.path.join(
                        output_directory,
                        os.path.relpath(root, destination_volume_path),
                        file_name,
                    )
                    destination_fs.upload_file_to_storage(
                        source_path=source_path, destination_path=destination_path
                    )
        except ConnectorError as e:
            raise UnstractFSException(core_err=e) from e

    def insert_into_db(self, input_file_path: str, error: str | None = None) -> None:
        """Insert data into the database."""
        connector_instance: ConnectorInstance = self.endpoint.connector_instance
        connector_settings: dict[str, Any] = connector_instance.connector_metadata
        destination_configurations: dict[str, Any] = self.endpoint.configuration
        table_name: str = str(destination_configurations.get(DestinationKey.TABLE))
        include_agent: bool = bool(
            destination_configurations.get(DestinationKey.INCLUDE_AGENT, False)
        )
        include_timestamp = bool(
            destination_configurations.get(DestinationKey.INCLUDE_TIMESTAMP, False)
        )
        agent_name = str(destination_configurations.get(DestinationKey.AGENT_NAME))
        column_mode = str(destination_configurations.get(DestinationKey.COLUMN_MODE))
        single_column_name = str(
            destination_configurations.get(DestinationKey.SINGLE_COLUMN_NAME, "data")
        )
        file_path_name = str(
            destination_configurations.get(DestinationKey.FILE_PATH, "file_path")
        )
        execution_id_name = str(
            destination_configurations.get(DestinationKey.EXECUTION_ID, "execution_id")
        )

        data = self.get_tool_execution_result() if not error else None
        metadata = self.get_combined_metadata()

        if not data and not error:
            logger.info("No data obtained from tool to insert into destination DB.")
            return

        # Log when we're proceeding with error insertion
        if error and not data:
            logger.info(
                f"Proceeding with error record insertion for {input_file_path}: {error}"
            )

        if isinstance(data, dict):
            data.pop("metadata", None)

        db_class = DatabaseUtils.get_db_class(
            connector_id=connector_instance.connector_id,
            connector_settings=connector_settings,
        )

        engine = db_class.get_engine()
        table_info = db_class.get_information_schema(table_name=table_name)

        logger.info(
            f"destination connector table_name: {table_name} with table_info: {table_info}"
        )

        if table_info:
            if db_class.has_no_metadata(table_info=table_info):
                table_info = DatabaseUtils.migrate_table_to_v2(
                    db_class=db_class,
                    engine=engine,
                    table_name=table_name,
                    column_name=single_column_name,
                )

        values = DatabaseUtils.get_columns_and_values(
            column_mode_str=column_mode,
            data=data,
            metadata=metadata,
            include_timestamp=include_timestamp,
            include_agent=include_agent,
            agent_name=agent_name,
            single_column_name=single_column_name,
            file_path_name=file_path_name,
            execution_id_name=execution_id_name,
            table_info=table_info,
            file_path=input_file_path,
            execution_id=self.execution_id,
            error=error,
        )

        logger.debug(f"destination.py values: {values}")

        try:
            # Reuse the same db_class and engine created earlier

            DatabaseUtils.create_table_if_not_exists(
                db_class=db_class,
                engine=engine,
                table_name=table_name,
                database_entry=values,
            )

            sql_columns_and_values = DatabaseUtils.get_sql_query_data(
                conn_cls=db_class,
                table_name=table_name,
                values=values,
            )
            logger.info("destination.py sql_columns_and_values", sql_columns_and_values)
            DatabaseUtils.execute_write_query(
                db_class=db_class,
                engine=engine,
                table_name=table_name,
                sql_keys=list(sql_columns_and_values.keys()),
                sql_values=list(sql_columns_and_values.values()),
            )

        except ConnectorError as e:
            error_msg = f"Database connection failed for {input_file_path}: {str(e)}"
            logger.error(error_msg)
            raise
        except Exception as e:
            error_msg = (
                f"Failed to insert data into database for {input_file_path}: {str(e)}"
            )
            logger.error(error_msg)
            raise
        finally:
            self._close_engine(engine, input_file_path)

    def _close_engine(self, engine: Any, input_file_path: str) -> None:
        """Safely close database engine."""
        if engine:
            try:
                engine.close()
            except Exception as e:
                logger.error(
                    f"Failed to close database engine for {input_file_path}: {str(e)}"
                )

    def _handle_api_result(
        self,
        file_name: str,
        error: str | None = None,
        result: str | None = None,
        metadata: dict[str, Any] | None = None,
    ) -> None:
        """Handle the API result.

        This method is responsible for handling the API result.
        It appends the file name and result to the 'results' list for API resp.

        Args:
            file_name (str): The name of the file.
            result (Optional[str], optional): The result of the API call.
                Defaults to None.

        Returns:
            None
        """
        api_result: dict[str, Any] = {
            "file": file_name,
            "file_execution_id": self.file_execution_id,
        }
        if error:
            api_result.update(
                {"status": ApiDeploymentResultStatus.FAILED, "error": error}
            )
        else:
            if result:
                api_result.update(
                    {
                        "status": ApiDeploymentResultStatus.SUCCESS,
                        "result": result,
                        "metadata": metadata,
                    }
                )
            else:
                api_result.update(
                    {"status": ApiDeploymentResultStatus.SUCCESS, "result": ""}
                )
        self.update_api_results(api_result)

    def parse_string(self, original_string: str) -> Any:
        """Parse the given string, attempting to evaluate it as a Python
        literal.
        ex: a json string to dict method
        Parameters:
        - original_string (str): The input string to be parsed.

        Returns:
        - Any: The parsed result. If the string can be evaluated as a Python
          literal, the result of the evaluation is returned.
          If not, the original string is returned unchanged.

        Note:
        This function uses `ast.literal_eval` to attempt parsing the string as a
        Python literal. If parsing fails due to a SyntaxError or ValueError,
        the original string is returned.

        Example:
        >>> parser.parse_string("42")
        42
        >>> parser.parse_string("[1, 2, 3]")
        [1, 2, 3]
        >>> parser.parse_string("Hello, World!")
        'Hello, World!'
        """
        try:
            # Try to evaluate as a Python literal
            python_literal = ast.literal_eval(original_string)
            return python_literal
        except (SyntaxError, ValueError):
            # If evaluating as a Python literal fails,
            # assume it's a plain string
            return original_string

    def get_tool_execution_result(
        self, file_history: FileHistory | None = None
    ) -> Any | None:
        """Get result data from the output file.

        Returns:
            Union[dict[str, Any], str]: Result data.
        """
        return self.get_tool_execution_result_from_metadata(file_history=file_history)

    def get_tool_execution_result_from_metadata(
        self, file_history: FileHistory | None = None
    ) -> Any | None:
        """Get result data from the output file.

        Returns:
            Union[dict[str, Any], str]: Result data.
        """
        if file_history and file_history.result:
            return self.parse_string(file_history.result)
        output_file = self.infile
        metadata: dict[str, Any] = self.get_workflow_metadata()
        output_type = self.get_output_type(metadata)
        result: dict[str, Any] | str = ""
        file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
        file_storage = file_system.get_file_storage()
        try:
            # TODO: SDK handles validation; consider removing here.
            file_type = file_storage.mime_type(path=output_file)
            if output_type == ToolOutputType.JSON:
                if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
                    msg = f"Expected tool output type: JSON, got: '{file_type}'"
                    logger.error(msg)
                    raise ToolOutputTypeMismatch(detail=msg)
                file_content = file_storage.read(output_file, mode="r")
                result = json.loads(file_content)
            elif output_type == ToolOutputType.TXT:
                if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
                    msg = f"Expected tool output type: TXT, got: '{file_type}'"
                    logger.error(msg)
                    raise ToolOutputTypeMismatch(detail=msg)
                file_content = file_storage.read(output_file, mode="r")
                result = file_content.encode("utf-8").decode("unicode-escape")
            else:
                raise InvalidToolOutputType()
        except (FileNotFoundError, json.JSONDecodeError) as err:
            msg = f"Error while getting result from the tool: {err}"
            logger.error(msg)
            raise APIException(detail=msg)

        return result

    def has_valid_metadata(self, metadata: Any) -> bool:
        # Check if metadata is not None and is either a non-empty dict or valid string
        if metadata is None:
            return False

        # Handle dict metadata (which is valid and contains extracted_text)
        if isinstance(metadata, dict):
            return bool(metadata)  # Return True if dict is not empty

        # Handle string metadata
        if isinstance(metadata, str):
            if metadata.strip().lower() == "none" or not metadata.strip():
                return False
            return True

        # For other types, consider them valid if they're truthy
        return bool(metadata)

    def get_metadata(
        self, file_history: FileHistory | None = None
    ) -> dict[str, Any] | None:
        """Get metadata from the output file.

        Returns:
            Union[dict[str, Any], str]: Metadata.
        """
        if file_history:
            if self.has_valid_metadata(file_history.metadata):
                return self.parse_string(file_history.metadata)
            else:
                return None
        metadata: dict[str, Any] = self.get_workflow_metadata()
        return metadata

    def get_combined_metadata(self) -> dict[str, Any]:
        """Get combined workflow and usage metadata.

        Returns:
            dict[str, Any]: Combined metadata including workflow and usage data.
        """
        # Get workflow metadata
        workflow_metadata = self.get_metadata()

        # Get file_execution_id from metadata
        file_execution_id = workflow_metadata.get("file_execution_id")
        if not file_execution_id:
            return workflow_metadata

        usage_metadata = UsageHelper.get_aggregated_token_count(file_execution_id)

        # Combine both metadata
        workflow_metadata["usage"] = usage_metadata
        workflow_metadata["total_pages_processed"] = (
            UsageHelper.get_aggregated_pages_processed(run_id=file_execution_id)
        )

        return workflow_metadata

    def delete_file_execution_directory(self) -> None:
        """Delete the file execution directory.

        Returns:
            None
        """
        file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
        file_storage = file_system.get_file_storage()
        if self.file_execution_dir and file_storage.exists(self.file_execution_dir):
            file_storage.rm(self.file_execution_dir, recursive=True)

    @classmethod
    def delete_execution_and_api_storage_dir(
        cls, workflow_id: str, execution_id: str
    ) -> None:
        """Delete the execution and api storage directories.

        Returns:
            None
        """
        cls.delete_execution_directory(workflow_id, execution_id)
        cls.delete_api_storage_dir(workflow_id, execution_id)

    @classmethod
    def delete_api_storage_dir(cls, workflow_id: str, execution_id: str) -> None:
        """Delete the api storage path.

        Returns:
            None
        """
        api_storage_dir = cls.get_api_storage_dir_path(
            workflow_id=workflow_id, execution_id=execution_id
        )
        file_system = FileSystem(FileStorageType.API_EXECUTION)
        file_storage = file_system.get_file_storage()
        if file_storage.exists(api_storage_dir):
            file_storage.rm(api_storage_dir, recursive=True)
            logger.info(f"API storage directory deleted: {api_storage_dir}")

    @classmethod
    def delete_execution_directory(cls, workflow_id: str, execution_id: str) -> None:
        """Delete the execution directory.

        Returns:
            None
        """
        execution_dir = cls.get_execution_dir_path(
            workflow_id=workflow_id, execution_id=execution_id
        )
        file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
        file_storage = file_system.get_file_storage()
        if file_storage.exists(execution_dir):
            file_storage.rm(execution_dir, recursive=True)
            logger.info(f"Execution directory deleted: {execution_dir}")

    @classmethod
    def create_endpoint_for_workflow(
        cls,
        workflow: Workflow,
    ) -> None:
        """Create a workflow endpoint for the destination.

        Args:
            workflow (Workflow): Workflow for which the endpoint is created.
        """
        endpoint = WorkflowEndpoint(
            workflow=workflow,
            endpoint_type=WorkflowEndpoint.EndpointType.DESTINATION,
        )
        endpoint.save()

    @classmethod
    def get_json_schema_for_database(cls) -> dict[str, Any]:
        """Get JSON schema for the database.

        Returns:
            dict[str, Any]: JSON schema for the database.
        """
        schema_path = os.path.join(os.path.dirname(__file__), "static", "dest", "db.json")
        return cls.get_json_schema(file_path=schema_path)

    @classmethod
    def get_json_schema_for_file_system(cls) -> dict[str, Any]:
        """Get JSON schema for the file system.

        Returns:
            dict[str, Any]: JSON schema for the file system.
        """
        schema_path = os.path.join(
            os.path.dirname(__file__), "static", "dest", "file.json"
        )
        return cls.get_json_schema(file_path=schema_path)

    @classmethod
    def get_json_schema_for_api(cls) -> dict[str, Any]:
        """Json schema for api.

        Returns:
            dict[str, Any]: _description_
        """
        schema_path = os.path.join(
            os.path.dirname(__file__), "static", "dest", "api.json"
        )
        return cls.get_json_schema(file_path=schema_path)

    def get_config(self) -> DestinationConfig:
        """Get serializable configuration for the destination connector.

        Returns:
            DestinationConfig: Configuration containing all necessary data to reconstruct the connector
        """
        return DestinationConfig(
            workflow_id=self.workflow.id,
            execution_id=self.execution_id,
            use_file_history=self.use_file_history,
            hitl_queue_name=self.hitl_queue_name,
            packet_id=self.packet_id,
        )

    @classmethod
    def from_config(
        cls, workflow_log: WorkflowLog, config: DestinationConfig
    ) -> "DestinationConnector":
        """Create a DestinationConnector instance from configuration.

        Args:
            config (DestinationConfig): Configuration containing all necessary data to reconstruct the connector

        Returns:
            DestinationConnector: New instance
        """
        logger.info(
            f"Creating DestinationConnector from config: hitl_queue_name={config.hitl_queue_name}"
        )
        # Reconstruct workflow
        workflow = Workflow.objects.get(id=config.workflow_id)

        # Create destination connector instance
        destination = cls(
            workflow=workflow,
            execution_id=config.execution_id,
            workflow_log=workflow_log,
            use_file_history=config.use_file_history,
            file_execution_id=config.file_execution_id,
            hitl_queue_name=config.hitl_queue_name,
            packet_id=config.packet_id,
        )

        return destination

    def _get_review_queue_name(self) -> str:
        """Generate review queue name with optional HITL override for manual review processing.

        Returns:
            str: Queue name in the appropriate format:
                - Custom HITL queue: review_queue_{org}_{workflow_id}:{hitl_queue_name}
                - Standard queue: review_queue_{org}_{workflow_id}
        """
        logger.debug(f"Queue naming - hitl_queue_name={self.hitl_queue_name}")

        # Base queue format: review_queue_{org}_{workflow_id}
        base_queue_name = f"review_queue_{self.organization_id}_{str(self.workflow_id)}"

        if self.hitl_queue_name:
            # Custom HITL queue with user-specified name
            q_name = f"{base_queue_name}:{self.hitl_queue_name}"
            logger.debug(f"Using custom HITL queue: {q_name}")
        else:
            # Standard queue format for workflow-based processing
            q_name = base_queue_name
            logger.debug(f"Using standard queue name: {q_name}")

        return q_name

    def _push_to_queue(
        self,
        file_name: str,
        workflow: Workflow,
        result: str | None = None,
        input_file_path: str | None = None,
        meta_data: dict[str, Any] | None = None,
        file_execution_id: str = None,
    ) -> None:
        """Handle the Manual Review QUEUE result.

        This method is responsible for pushing the input file and result to
        review queue.

        Args:
            file_name (str): The name of the file.
            workflow (Workflow): The workflow object containing
            details about the workflow.
            result (Optional[str], optional): The result of the API call.
                Defaults to None.
            input_file_path (Optional[str], optional):
            The path to the input file.
                Defaults to None.
            meta_data (Optional[dict[str, Any]], optional):
                A dictionary containing additional
                metadata related to the file. Defaults to None.

        Returns:
            None
        """
        # Handle missing result for packet processing
        if not result:
            if not self.packet_id:
                return
            result = json.dumps({"status": "pending", "message": "Awaiting processing"})

        # Delegate to appropriate handler based on deployment type
        if self.is_api:
            self._push_to_queue_for_api_deployment(
                file_name, result, input_file_path, file_execution_id, meta_data
            )
        else:
            self._push_to_queue_for_connector(
                file_name,
                workflow,
                result,
                input_file_path,
                file_execution_id,
                meta_data,
            )

    def _enqueue_to_packet_or_regular_queue(
        self,
        file_name: str,
        queue_result: dict[str, Any],
        queue_result_json: str,
        q_name: str,
        ttl_seconds: int | None = None,
    ) -> None:
        """Route to packet queue or regular queue based on packet_id.

        Args:
            file_name: Name of the file being queued
            queue_result: Queue result dictionary
            queue_result_json: JSON string of queue result
            q_name: Queue name for regular queue
            ttl_seconds: TTL in seconds (optional, for regular queue)
        """
        # Get organization_id for proper HITL queue filtering
        organization_id = UserContext.get_organization_identifier()

        # Get HITL backend queue instance with organization context
        conn = QueueUtils.get_queue_inst(
            {"use_hitl_backend": True, "organization_id": organization_id}
        )

        # Check if this is packet processing
        if self.packet_id:
            # Route to packet queue (Enterprise feature)
            logger.info(f"Routing {file_name} to packet queue {self.packet_id}")
            success = conn.enqueue_to_packet(
                packet_id=self.packet_id,
                queue_result=queue_result,
                organization_id=organization_id,
            )
            if not success:
                error_msg = f"Failed to push {file_name} to packet {self.packet_id}"
                logger.error(error_msg)
                raise RuntimeError(error_msg)
            logger.info(
                f"Pushed {file_name} to packet {self.packet_id} with organization_id {organization_id}"
            )
            return

        # Route to regular queue
        if ttl_seconds:
            conn.enqueue_with_ttl(
                queue_name=q_name, message=queue_result_json, ttl_seconds=ttl_seconds
            )
            logger.info(
                f"Pushed {file_name} to queue {q_name} with organization_id {organization_id} and TTL {ttl_seconds}"
            )
        else:
            conn.enqueue(queue_name=q_name, message=queue_result_json)
            logger.info(
                f"Pushed {file_name} to queue {q_name} with organization_id {organization_id}"
            )

    def _create_queue_result(
        self,
        file_name: str,
        result: str,
        file_content_base64: str,
        file_execution_id: str,
        meta_data: dict[str, Any] | None = None,
        ttl_seconds: int | None = None,
        hitl_reason: str | None = None,
    ) -> dict[str, Any]:
        """Create QueueResult dictionary.

        Args:
            file_name: Name of the file
            result: Processing result
            file_content_base64: Base64 encoded file content
            file_execution_id: File execution ID
            meta_data: Optional metadata
            ttl_seconds: Optional TTL in seconds
            hitl_reason: Optional reason why file was sent to HITL

        Returns:
            QueueResult as dictionary
        """
        whisper_hash = meta_data.get("whisper-hash") if meta_data else None
        extracted_text = meta_data.get("extracted_text") if meta_data else None

        queue_result_obj = QueueResult(
            file=file_name,
            status=QueueResultStatus.SUCCESS,
            result=result,
            workflow_id=str(self.workflow_id),
            file_content=file_content_base64,
            whisper_hash=whisper_hash,
            file_execution_id=file_execution_id,
            execution_id=self.execution_id,
            extracted_text=extracted_text,
            ttl_seconds=ttl_seconds,
            hitl_reason=hitl_reason,
        )
        return queue_result_obj.to_dict()

    def _push_to_queue_for_api_deployment(
        self,
        file_name: str,
        result: str,
        input_file_path: str,
        file_execution_id: str,
        meta_data: dict[str, Any] | None = None,
    ) -> None:
        """Handle queue push for API deployments.

        Args:
            file_name: Name of the file
            result: Processing result
            input_file_path: Path to input file
            file_execution_id: File execution ID
            meta_data: Optional metadata
        """
        file_content_base64 = self._read_file_content_for_queue(
            input_file_path, file_name
        )
        q_name = self._get_review_queue_name()
        ttl_seconds = WorkflowUtil.get_hitl_ttl_seconds(self.workflow)

        queue_result = self._create_queue_result(
            file_name=file_name,
            result=result,
            file_content_base64=file_content_base64,
            file_execution_id=file_execution_id,
            meta_data=meta_data,
            ttl_seconds=ttl_seconds,
        )

        queue_result_json = json.dumps(queue_result)
        self._enqueue_to_packet_or_regular_queue(
            file_name, queue_result, queue_result_json, q_name, ttl_seconds
        )

    def _push_to_queue_for_connector(
        self,
        file_name: str,
        workflow: Workflow,
        result: str,
        input_file_path: str,
        file_execution_id: str,
        meta_data: dict[str, Any] | None = None,
    ) -> None:
        """Handle queue push for connector-based deployments.

        Args:
            file_name: Name of the file
            workflow: Workflow object
            result: Processing result
            input_file_path: Path to input file
            file_execution_id: File execution ID
            meta_data: Optional metadata
        """
        connector = self.source_endpoint.connector_instance
        connector_settings = connector.connector_metadata

        source_fs = self.get_fsspec(
            settings=connector_settings, connector_id=connector.connector_id
        )

        with source_fs.open(input_file_path, "rb") as remote_file:
            file_content = remote_file.read()
            file_content_base64 = base64.b64encode(file_content).decode("utf-8")

        q_name = self._get_review_queue_name()
        ttl_seconds = WorkflowUtil.get_hitl_ttl_seconds(workflow)

        queue_result = self._create_queue_result(
            file_name=file_name,
            result=result,
            file_content_base64=file_content_base64,
            file_execution_id=file_execution_id,
            meta_data=meta_data,
            ttl_seconds=ttl_seconds,
        )

        queue_result_json = json.dumps(queue_result)

        # Validate JSON is not empty
        if not queue_result_json or queue_result_json.strip() == "":
            logger.error(f"Attempted to enqueue empty JSON with TTL for file {file_name}")
            raise ValueError("Cannot enqueue empty JSON message")

        self._enqueue_to_packet_or_regular_queue(
            file_name, queue_result, queue_result_json, q_name, ttl_seconds
        )

    def _read_file_content_for_queue(self, input_file_path: str, file_name: str) -> str:
        """Read and encode file content for queue message.

        Args:
            input_file_path: Path to the file to read
            file_name: Name of the file for logging purposes

        Returns:
            Base64 encoded file content

        Raises:
            APIException: If file cannot be read or doesn't exist
        """
        try:
            file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
            file_storage = file_system.get_file_storage()

            if not file_storage.exists(input_file_path):
                raise APIException(f"File not found: {input_file_path}")

            file_bytes = file_storage.read(input_file_path, mode="rb")
            if isinstance(file_bytes, str):
                file_bytes = file_bytes.encode("utf-8")
            return base64.b64encode(file_bytes).decode("utf-8")
        except Exception as e:
            logger.error(f"Failed to read file content for {file_name}: {e}")
            raise APIException(f"Failed to read file content for queue: {e}")