LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat: UN-2166 Passed optional container name from backend to runner (#1158)
Chandrasekharan Mc4fb6b82c327
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex 2464200c5..c03ffda81 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -231,9 +231,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):)raise WorkflowExecutionError(self.compilation_result["problems"][0])- def execute(- self, file_execution_id: str, file_name: str, single_step: bool = False- ) -> None:+ def execute(self, file_execution_id: str, single_step: bool = False) -> None:execution_type = ExecutionType.COMPLETEif single_step:execution_type = ExecutionType.STEP@@ -256,7 +254,6 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):try:self.execute_workflow(file_execution_id=file_execution_id,- file_name=file_name,execution_type=execution_type,)end_time = time.time()@@ -356,7 +353,13 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):)workflow_file_execution.update_status(ExecutionStatus.EXECUTING)- self.execute(file_execution_id, file_name, single_step)+ logger.info(+ f"Running execution: '{self.execution_id}', "+ f"file_execution_id: '{file_execution_id}', "+ f"file '{file_name}'"+ )++ self.execute(file_execution_id, single_step)self.publish_log(f"Tool executed successfully for '{file_name}'")self._handle_execution_type(execution_type)diff --git a/docker/scripts/pdm-lock-gen/pdm-lock.sh b/docker/scripts/pdm-lock-gen/pdm-lock.shindex bbc218de8..e0d7c2bcf 100755--- a/docker/scripts/pdm-lock-gen/pdm-lock.sh+++ b/docker/scripts/pdm-lock-gen/pdm-lock.sh@@ -79,7 +79,6 @@ directories=("platform-service""x2text-service""unstract/connectors"- "unstract/tool-sandbox")# If directories are passed as arguments, override the defaultdiff --git a/runner/src/unstract/runner/clients/docker.py b/runner/src/unstract/runner/clients/docker.pyindex 483bb16f1..8b5050206 100644--- a/runner/src/unstract/runner/clients/docker.py+++ b/runner/src/unstract/runner/clients/docker.py@@ -158,17 +158,24 @@ class Client(ContainerClientInterface):def get_container_run_config(self,command: list[str],- run_id: str,+ file_execution_id: str,+ container_name: Optional[str] = None,envs: Optional[dict[str, Any]] = None,auto_remove: bool = False,) -> dict[str, Any]:if envs is None:envs = {}mounts = []++ if not container_name:+ container_name = UnstractUtils.build_tool_container_name(+ tool_image=self.image_name,+ tool_version=self.image_tag,+ file_execution_id=file_execution_id,+ )+return {- "name": UnstractUtils.build_tool_container_name(- tool_image=self.image_name, tool_version=self.image_tag, run_id=run_id- ),+ "name": container_name,"image": self.get_image(),"command": command,"detach": True,diff --git a/runner/src/unstract/runner/clients/interface.py b/runner/src/unstract/runner/clients/interface.pyindex 9d0876ba7..2ad65fb42 100644--- a/runner/src/unstract/runner/clients/interface.py+++ b/runner/src/unstract/runner/clients/interface.py@@ -62,7 +62,8 @@ class ContainerClientInterface(ABC):def get_container_run_config(self,command: list[str],- run_id: str,+ file_execution_id: str,+ container_name: Optional[str] = None,envs: Optional[dict[str, Any]] = None,auto_remove: bool = False,) -> dict[str, Any]:diff --git a/runner/src/unstract/runner/clients/test_docker.py b/runner/src/unstract/runner/clients/test_docker.pyindex 42d99b56f..c275e15e0 100644--- a/runner/src/unstract/runner/clients/test_docker.py+++ b/runner/src/unstract/runner/clients/test_docker.py@@ -109,7 +109,7 @@ def test_get_image(docker_client, mocker):def test_get_container_run_config(docker_client, mocker):"""Test the get_container_run_config method."""command = ["echo", "hello"]- run_id = "run123"+ file_execution_id = "run123"mocker.patch.object(docker_client, "_Client__image_exists", return_value=True)mocker_normalize = mocker.patch(@@ -117,11 +117,13 @@ def test_get_container_run_config(docker_client, mocker):return_value="test-image",)config = docker_client.get_container_run_config(- command, run_id, envs={"KEY": "VALUE"}, auto_remove=True+ command, file_execution_id, envs={"KEY": "VALUE"}, auto_remove=True)mocker_normalize.assert_called_once_with(- tool_image="test-image", tool_version="latest", run_id=run_id+ tool_image="test-image",+ tool_version="latest",+ file_execution_id=file_execution_id,)assert config["name"] == "test-image"assert config["image"] == "test-image:latest"@@ -134,17 +136,21 @@ def test_get_container_run_config_without_mount(docker_client, mocker):"""Test the get_container_run_config method."""os.environ[Env.EXECUTION_DATA_DIR] = "/source"command = ["echo", "hello"]- run_id = "run123"+ file_execution_id = "run123"mocker.patch.object(docker_client, "_Client__image_exists", return_value=True)mocker_normalize = mocker.patch("unstract.core.utilities.UnstractUtils.build_tool_container_name",return_value="test-image",)- config = docker_client.get_container_run_config(command, run_id, auto_remove=True)+ config = docker_client.get_container_run_config(+ command, file_execution_id, auto_remove=True+ )mocker_normalize.assert_called_once_with(- tool_image="test-image", tool_version="latest", run_id=run_id+ tool_image="test-image",+ tool_version="latest",+ file_execution_id=file_execution_id,)assert config["name"] == "test-image"assert config["image"] == "test-image:latest"diff --git a/runner/src/unstract/runner/main.py b/runner/src/unstract/runner/main.pyindex 3caa480d4..43dd1dcb9 100644--- a/runner/src/unstract/runner/main.py+++ b/runner/src/unstract/runner/main.py@@ -28,17 +28,19 @@ def run_container() -> Optional[Any]:organization_id = data["organization_id"]workflow_id = data["workflow_id"]execution_id = data["execution_id"]- run_id = data["run_id"]+ file_execution_id = data["file_execution_id"]+ container_name = data["container_name"]settings = data["settings"]envs = data["envs"]messaging_channel = data["messaging_channel"]runner = UnstractRunner(image_name, image_tag, app)result = runner.run_container(+ container_name=container_name,organization_id=organization_id,workflow_id=workflow_id,execution_id=execution_id,- run_id=run_id,+ file_execution_id=file_execution_id,settings=settings,envs=envs,messaging_channel=messaging_channel,diff --git a/runner/src/unstract/runner/runner.py b/runner/src/unstract/runner/runner.pyindex a1e31b757..98a43023c 100644--- a/runner/src/unstract/runner/runner.py+++ b/runner/src/unstract/runner/runner.py@@ -39,7 +39,7 @@ class UnstractRunner:tool_instance_id: str,execution_id: str,organization_id: str,- run_id: str,+ file_execution_id: str,channel: Optional[str] = None,) -> None:for line in container.logs(follow=True):@@ -51,7 +51,7 @@ class UnstractRunner:channel=channel,execution_id=execution_id,organization_id=organization_id,- run_id=run_id,+ file_execution_id=file_execution_id,)def get_valid_log_message(self, log_message: str) -> Optional[dict[str, Any]]:@@ -76,7 +76,7 @@ class UnstractRunner:tool_instance_id: str,execution_id: str,organization_id: str,- run_id: str,+ file_execution_id: str,channel: Optional[str] = None,) -> Optional[dict[str, Any]]:log_dict = self.get_valid_log_message(log_message)@@ -99,7 +99,7 @@ class UnstractRunner:log_dict[LogFieldName.EXECUTION_ID] = execution_idlog_dict[LogFieldName.ORGANIZATION_ID] = organization_idlog_dict[LogFieldName.TIMESTAMP] = datetime.now(timezone.utc).timestamp()- log_dict[LogFieldName.FILE_EXECUTION_ID] = run_id+ log_dict[LogFieldName.FILE_EXECUTION_ID] = file_execution_id# Publish to channel of socket ioLogPublisher.publish(channel, log_dict)@@ -139,7 +139,7 @@ class UnstractRunner:"""command = command.upper()container_config = self.client.get_container_run_config(- command=["--command", command], run_id="", auto_remove=True+ command=["--command", command], file_execution_id="", auto_remove=True)container = None@@ -163,10 +163,11 @@ class UnstractRunner:organization_id: str,workflow_id: str,execution_id: str,- run_id: str,+ file_execution_id: str,settings: dict[str, Any],envs: dict[str, Any],messaging_channel: Optional[str] = None,+ container_name: Optional[str] = None,) -> Optional[Any]:"""RUN container With RUN Command.@@ -200,7 +201,8 @@ class UnstractRunner:"--log-level","DEBUG",],- run_id=run_id,+ file_execution_id=file_execution_id,+ container_name=container_name,envs=envs,)# Add labels to container for logging with Loki.@@ -217,7 +219,7 @@ class UnstractRunner:try:self.logger.info(f"Execution ID: {execution_id}, running docker "- f"container: {container_config.get('name')}"+ f"container: {container_name}")container: ContainerInterface = self.client.run_container(container_config)tool_instance_id = str(settings.get(ToolKey.TOOL_INSTANCE_ID))@@ -228,7 +230,7 @@ class UnstractRunner:channel=messaging_channel,execution_id=execution_id,organization_id=organization_id,- run_id=run_id,+ file_execution_id=file_execution_id,)except ToolRunException as te:self.logger.error(diff --git a/unstract/core/src/unstract/core/utilities.py b/unstract/core/src/unstract/core/utilities.pyindex 5c1a7c523..97b96a843 100644--- a/unstract/core/src/unstract/core/utilities.py+++ b/unstract/core/src/unstract/core/utilities.py@@ -30,12 +30,21 @@ class UnstractUtils:@staticmethoddef build_tool_container_name(- tool_image: str, tool_version: str, run_id: str+ tool_image: str,+ tool_version: str,+ file_execution_id: str,+ retry_count: Optional[int] = None,) -> str:tool_name = tool_image.split("/")[-1]- # TODO: Add execution attempt to better track instead of uuid- short_uuid = uuid.uuid4().hex[:6] # To avoid duplicate name collision- container_name = f"{tool_name}-{tool_version}-{short_uuid}-{run_id}"+ # To avoid duplicate name collision+ if retry_count:+ unique_suffix = retry_count+ else:+ unique_suffix = uuid.uuid4().hex[:6]++ container_name = (+ f"{tool_name}-{tool_version}-{unique_suffix}-{file_execution_id}"+ )# To support limits of container clients like K8sif len(container_name) > 63:diff --git a/unstract/tool-sandbox/pyproject.toml b/unstract/tool-sandbox/pyproject.tomlindex d8bcab20f..b0f37f06d 100644--- a/unstract/tool-sandbox/pyproject.toml+++ b/unstract/tool-sandbox/pyproject.toml@@ -10,7 +10,18 @@ authors = [{name = "Zipstack Inc.", email = "[email protected]"},]dependencies = [- "requests==2.31.0"+ "requests==2.31.0",+ # ! IMPORTANT!+ # Local dependencies usually need to be added as:+ # https://pdm-project.org/latest/usage/dependency/#local-dependencies+ #+ # However, local dependencies which are not direct depedency of main project+ # appear as absolute paths in pdm.lock of main project, making it impossible+ # to check in the lock file.+ #+ # Hence instead, add the dependencies without version constraints where the+ # assumption is they are added as direct dependencies in main project itself.+ "unstract-core",]requires-python = ">=3.9"readme = "README.md"diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.pyindex d52a95443..39a3887bf 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/helper.py@@ -6,6 +6,8 @@ from typing import Any, Optional, Unionimport requestsfrom unstract.tool_sandbox.constants import UnstractRunner+from unstract.core.utilities import UnstractUtils+logger = logging.getLogger(__name__)@@ -69,10 +71,11 @@ class ToolSandboxHelper:def call_tool_handler(self,- run_id: str,+ file_execution_id: str,image_name: str,image_tag: str,settings: dict[str, Any],+ retry_count: Optional[int] = None,) -> Optional[dict[str, Any]]:"""Calling unstract runner to run the required tool.@@ -87,10 +90,7 @@ class ToolSandboxHelper:"""url = f"{self.base_url}{UnstractRunner.RUN_API_ENDPOINT}"data = self.create_tool_request_data(- run_id,- image_name,- image_tag,- settings,+ file_execution_id, image_name, image_tag, settings, retry_count)response = requests.post(url, json=data)@@ -110,18 +110,26 @@ class ToolSandboxHelper:def create_tool_request_data(self,- run_id: str,+ file_execution_id: str,image_name: str,image_tag: str,settings: dict[str, Any],+ retry_count: Optional[int] = None,) -> dict[str, Any]:+ container_name = UnstractUtils.build_tool_container_name(+ tool_image=image_name,+ tool_version=image_tag,+ file_execution_id=file_execution_id,+ retry_count=retry_count,+ )data = {"image_name": image_name,"image_tag": image_tag,"organization_id": self.organization_id,"workflow_id": self.workflow_id,"execution_id": self.execution_id,- "run_id": run_id,+ "file_execution_id": file_execution_id,+ "container_name": container_name,"settings": settings,"envs": self.envs,"messaging_channel": self.messaging_channel,diff --git a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.pyindex c9a277843..ad85c29d6 100644--- a/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py+++ b/unstract/tool-sandbox/src/unstract/tool_sandbox/tool_sandbox.py@@ -93,10 +93,13 @@ class ToolSandbox:)return result- def run_tool(self, run_id: str) -> Optional[dict[str, Any]]:+ def run_tool(+ self, file_execution_id: str, retry_count: Optional[int] = None+ ) -> Optional[dict[str, Any]]:return self.helper.call_tool_handler( # type: ignore- run_id,+ file_execution_id,self.image_name,self.image_tag,self.settings,+ retry_count,)diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex 60ffccb85..c1d789c9b 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -174,21 +174,21 @@ class ToolsUtils:def run_tool(self,- run_id: str,+ file_execution_id: str,tool_sandbox: ToolSandbox,) -> Any:- return self.run_tool_with_retry(run_id, tool_sandbox)+ return self.run_tool_with_retry(file_execution_id, tool_sandbox)def run_tool_with_retry(self,- run_id: str,+ file_execution_id: str,tool_sandbox: ToolSandbox,max_retries: int = ToolExecution.MAXIMUM_RETRY,) -> Any:error: Optional[dict[str, Any]] = Nonefor retry_count in range(max_retries):try:- response = tool_sandbox.run_tool(run_id)+ response = tool_sandbox.run_tool(file_execution_id, retry_count)if response:return responselogger.warning(diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 39b94fd53..a11971c65 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -24,7 +24,6 @@ from unstract.workflow_execution.execution_file_handler import ExecutionFileHandfrom unstract.workflow_execution.tools_utils import ToolsUtilsfrom unstract.core.pubsub_helper import LogPublisher-from unstract.core.utilities import UnstractUtilslogger = logging.getLogger(__name__)@@ -140,14 +139,13 @@ class WorkflowExecutionService:logger.info(f"Execution {self.execution_id}: Build completed")def execute_workflow(- self, file_execution_id: str, file_name: str, execution_type: ExecutionType+ self, file_execution_id: str, execution_type: ExecutionType) -> None:"""Executes the complete workflow by running each tools one by one.Returns the result from final tool in a dictionary.Args:file_execution_id (str): UUID for a single run of a file- file_name (str): Name of the file to processexecution_type (ExecutionType): STEP or COMPLETERaises:@@ -168,16 +166,6 @@ class WorkflowExecutionService:# only. While supporting more tools in a workflow, correct the tool container# name to avoid conflicts.for step, sandbox in enumerate(self.tool_sandboxes):- container_name = UnstractUtils.build_tool_container_name(- tool_image=sandbox.image_name,- tool_version=sandbox.image_tag,- run_id=file_execution_id,- )- logger.info(- f"Running execution: '{self.execution_id}', "- f"tool: '{sandbox.image_name}:{sandbox.image_tag}', "- f"file '{file_name}', container: '{container_name}'"- )self._execute_step(step=step,sandbox=sandbox,@@ -224,7 +212,7 @@ class WorkflowExecutionService:component=tool_instance_id,)result = self.tool_utils.run_tool(- run_id=self.file_execution_id, tool_sandbox=sandbox+ file_execution_id=self.file_execution_id, tool_sandbox=sandbox)if result and result.get("error"):raise ToolOutputNotFoundException(result.get("error"))