LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: Passed EXECUTION_BY_TOOL env to tools from worker (#683)
Chandrasekharan M891b97e7e869
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex a31b9847a..7c855e4a0 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -40,6 +40,7 @@ from workflow_manager.workflow.models.workflow import Workflowlogger = logging.getLogger(__name__)+# TODO: Inherit from SourceConnector for different sources - File, API .etc.class SourceConnector(BaseConnector):"""A class representing a source connector for a workflow.@@ -351,6 +352,7 @@ class SourceConnector(BaseConnector):file_hash=file_hash,)+ # TODO: Get file count from len of dict instead of returning tupledef list_files_from_source(self, file_hashes: dict[str, FileHash] = {}) -> tuple[dict[str, FileHash], int]:@@ -559,11 +561,11 @@ class SourceConnector(BaseConnector):"""Add input file to api storage.Args:- workflow_id (str): workflow id- execution_id (str): execution_id- file_objs (list[UploadedFile]): api file objects+ workflow_id (str): UUID of the worklfow+ execution_id (str): UUID of the execution+ file_objs (list[UploadedFile]): List of uploaded filesreturns:- dict[str, FileHash]: file hashes+ dict[str, FileHash]: Dict containing file name and its corresponding hash"""api_storage_dir = cls.get_api_storage_dir_path(workflow_id=workflow_id, execution_id=execution_iddiff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.pyindex 57ef8d3f5..d9283fa7a 100644--- a/backend/workflow_manager/endpoint_v2/source.py+++ b/backend/workflow_manager/endpoint_v2/source.py@@ -37,6 +37,7 @@ from workflow_manager.workflow_v2.models.workflow import Workflowlogger = logging.getLogger(__name__)+# TODO: Inherit from SourceConnector for different sources - File, API .etc.class SourceConnector(BaseConnector):"""A class representing a source connector for a workflow.@@ -404,9 +405,11 @@ class SourceConnector(BaseConnector):"""Add input file to api storage.Args:- workflow_id (str): workflow id- execution_id (str): execution_id- file_objs (list[UploadedFile]): api file objects+ workflow_id (str): UUID of the worklfow+ execution_id (str): UUID of the execution+ file_objs (list[UploadedFile]): List of uploaded files+ Returns:+ dict[str, FileHash]: Dict containing file name and its corresponding hash"""api_storage_dir = cls.get_api_storage_dir_path(workflow_id=workflow_id, execution_id=execution_iddiff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.pyindex 5bf1372ac..172d12769 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py@@ -20,6 +20,7 @@ class ToolRuntimeVariable:X2TEXT_PORT = "X2TEXT_PORT"ADAPTER_LLMW_POLL_INTERVAL = "ADAPTER_LLMW_POLL_INTERVAL"ADAPTER_LLMW_MAX_POLLS = "ADAPTER_LLMW_MAX_POLLS"+ EXECUTION_BY_TOOL = "EXECUTION_BY_TOOL"class WorkflowFileType:diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex f76ca885c..705225b5e 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -198,18 +198,13 @@ class ToolsUtils:)return None- def get_tool_environment_variables(- self, project_settings: Optional[dict[str, Any]] = None- ) -> dict[str, Any]:+ def get_tool_environment_variables(self) -> dict[str, Any]:"""Obtain a dictionary of env variables required by a tool.This combines the user defined envs (coming from project_settings)as well as the env variables that are platform specific intoone key-value store.- Args:- project_settings (Optional[dict[str, Any]]): User defined settings-Returns:dict[str, Any]: Dict of env variables for a tool"""@@ -221,15 +216,14 @@ class ToolsUtils:ToolRV.PROMPT_PORT: self.prompt_port,ToolRV.X2TEXT_HOST: self.x2text_host,ToolRV.X2TEXT_PORT: self.x2text_port,+ ToolRV.EXECUTION_BY_TOOL: True,}# For async LLM Whisperer extractionif self.llmw_poll_interval:platform_vars[ToolRV.ADAPTER_LLMW_POLL_INTERVAL] = self.llmw_poll_intervalif self.llmw_max_polls:platform_vars[ToolRV.ADAPTER_LLMW_MAX_POLLS] = self.llmw_max_polls- if not project_settings:- project_settings = {}- return {**project_settings, **platform_vars}+ return platform_vars@staticmethoddef get_env(env_key: str, raise_exception: bool = False) -> Optional[str]: