Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

fix: Passed EXECUTION_BY_TOOL env to tools from worker (#683)

Chandrasekharan M891b97e7e869
diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.py
index a31b9847a..7c855e4a0 100644
--- a/backend/workflow_manager/endpoint/source.py
+++ b/backend/workflow_manager/endpoint/source.py
@@ -40,6 +40,7 @@ from workflow_manager.workflow.models.workflow import Workflow
logger = logging.getLogger(__name__)
+# TODO: Inherit from SourceConnector for different sources - File, API .etc.
class SourceConnector(BaseConnector):
"""A class representing a source connector for a workflow.
@@ -351,6 +352,7 @@ class SourceConnector(BaseConnector):
file_hash=file_hash,
)
+ # TODO: Get file count from len of dict instead of returning tuple
def list_files_from_source(
self, file_hashes: dict[str, FileHash] = {}
) -> tuple[dict[str, FileHash], int]:
@@ -559,11 +561,11 @@ class SourceConnector(BaseConnector):
"""Add input file to api storage.
Args:
- workflow_id (str): workflow id
- execution_id (str): execution_id
- file_objs (list[UploadedFile]): api file objects
+ workflow_id (str): UUID of the worklfow
+ execution_id (str): UUID of the execution
+ file_objs (list[UploadedFile]): List of uploaded files
returns:
- dict[str, FileHash]: file hashes
+ dict[str, FileHash]: Dict containing file name and its corresponding hash
"""
api_storage_dir = cls.get_api_storage_dir_path(
workflow_id=workflow_id, execution_id=execution_id
diff --git a/backend/workflow_manager/endpoint_v2/source.py b/backend/workflow_manager/endpoint_v2/source.py
index 57ef8d3f5..d9283fa7a 100644
--- a/backend/workflow_manager/endpoint_v2/source.py
+++ b/backend/workflow_manager/endpoint_v2/source.py
@@ -37,6 +37,7 @@ from workflow_manager.workflow_v2.models.workflow import Workflow
logger = logging.getLogger(__name__)
+# TODO: Inherit from SourceConnector for different sources - File, API .etc.
class SourceConnector(BaseConnector):
"""A class representing a source connector for a workflow.
@@ -404,9 +405,11 @@ class SourceConnector(BaseConnector):
"""Add input file to api storage.
Args:
- workflow_id (str): workflow id
- execution_id (str): execution_id
- file_objs (list[UploadedFile]): api file objects
+ workflow_id (str): UUID of the worklfow
+ execution_id (str): UUID of the execution
+ file_objs (list[UploadedFile]): List of uploaded files
+ Returns:
+ dict[str, FileHash]: Dict containing file name and its corresponding hash
"""
api_storage_dir = cls.get_api_storage_dir_path(
workflow_id=workflow_id, execution_id=execution_id
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
index 5bf1372ac..172d12769 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/constants.py
@@ -20,6 +20,7 @@ class ToolRuntimeVariable:
X2TEXT_PORT = "X2TEXT_PORT"
ADAPTER_LLMW_POLL_INTERVAL = "ADAPTER_LLMW_POLL_INTERVAL"
ADAPTER_LLMW_MAX_POLLS = "ADAPTER_LLMW_MAX_POLLS"
+ EXECUTION_BY_TOOL = "EXECUTION_BY_TOOL"
class WorkflowFileType:
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index f76ca885c..705225b5e 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -198,18 +198,13 @@ class ToolsUtils:
)
return None
- def get_tool_environment_variables(
- self, project_settings: Optional[dict[str, Any]] = None
- ) -> dict[str, Any]:
+ def get_tool_environment_variables(self) -> dict[str, Any]:
"""Obtain a dictionary of env variables required by a tool.
This combines the user defined envs (coming from project_settings)
as well as the env variables that are platform specific into
one key-value store.
- Args:
- project_settings (Optional[dict[str, Any]]): User defined settings
-
Returns:
dict[str, Any]: Dict of env variables for a tool
"""
@@ -221,15 +216,14 @@ class ToolsUtils:
ToolRV.PROMPT_PORT: self.prompt_port,
ToolRV.X2TEXT_HOST: self.x2text_host,
ToolRV.X2TEXT_PORT: self.x2text_port,
+ ToolRV.EXECUTION_BY_TOOL: True,
}
# For async LLM Whisperer extraction
if self.llmw_poll_interval:
platform_vars[ToolRV.ADAPTER_LLMW_POLL_INTERVAL] = self.llmw_poll_interval
if self.llmw_max_polls:
platform_vars[ToolRV.ADAPTER_LLMW_MAX_POLLS] = self.llmw_max_polls
- if not project_settings:
- project_settings = {}
- return {**project_settings, **platform_vars}
+ return platform_vars
@staticmethod
def get_env(env_key: str, raise_exception: bool = False) -> Optional[str]: