LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
[MISC] Tool sidecar lock updated and env default added (#1375)
Chandrasekharan M57d28416cbc0
diff --git a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py b/tool-sidecar/src/unstract/tool_sidecar/log_processor.pyindex 532d124be..86ade0cd7 100644--- a/tool-sidecar/src/unstract/tool_sidecar/log_processor.py+++ b/tool-sidecar/src/unstract/tool_sidecar/log_processor.py@@ -255,7 +255,7 @@ def main():# Needed for Kombu (used from unstract-core)celery_broker_base_url = os.getenv(Env.CELERY_BROKER_BASE_URL)celery_broker_user = os.getenv(Env.CELERY_BROKER_USER)- celery_broker_pass = os.getenv(Env.CELERY_BROKER_PASS)+ celery_broker_pass = os.getenv(Env.CELERY_BROKER_PASS, "")# Get execution parameters from environmenttool_instance_id = os.getenv(Env.TOOL_INSTANCE_ID)diff --git a/tool-sidecar/uv.lock b/tool-sidecar/uv.lockindex cab8d512c..ab174751c 100644--- a/tool-sidecar/uv.lock+++ b/tool-sidecar/uv.lock@@ -14,6 +14,20 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/26/99/fc813cd978842c26c82534010ea849eee9ab3a13ea2b74e95cb9c99e747b/amqp-5.3.1-py3-none-any.whl", hash = "sha256:43b3319e1b4e7d1251833a93d672b4af1e40f3d632d479b98661a95f117880a2", size = 50944 },]+[[package]]+name = "anyio"+version = "4.9.0"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "idna" },+ { name = "sniffio" },+ { name = "typing-extensions" },+]+sdist = { url = "https://files.pythonhosted.org/packages/95/7d/4c1bd541d4dffa1b52bd83fb8527089e097a106fc90b467a7313b105f840/anyio-4.9.0.tar.gz", hash = "sha256:673c0c244e15788651a4ff38710fea9675823028a6f08a5eda409e0c9840a028", size = 190949 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/a1/ee/48ca1a7c89ffec8b6a0c5d02b89c305671d5ffd8d3c94acf8b8c408575bb/anyio-4.9.0-py3-none-any.whl", hash = "sha256:9f76d541cad6e36af7beb62e978876f3b41e3e04f2c1fbf0884604c0a9c4d93c", size = 100916 },+]+[[package]]name = "certifi"version = "2025.1.31"@@ -87,6 +101,43 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/fd/7c/1e429c5fb26122055d10ff9a1d754790fb067d83c633ff69eddcf8e3614b/grpcio-1.71.0-cp312-cp312-win_amd64.whl", hash = "sha256:652350609332de6dac4ece254e5d7e1ff834e203d6afb769601f286886f6f3a8", size = 4272191 },]+[[package]]+name = "h11"+version = "0.16.0"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515 },+]++[[package]]+name = "httpcore"+version = "1.0.9"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "certifi" },+ { name = "h11" },+]+sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784 },+]++[[package]]+name = "httpx"+version = "0.28.1"+source = { registry = "https://pypi.org/simple" }+dependencies = [+ { name = "anyio" },+ { name = "certifi" },+ { name = "httpcore" },+ { name = "idna" },+]+sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517 },+]+[[package]]name = "idna"version = "3.10"@@ -110,15 +161,17 @@ wheels = [[[package]]name = "kombu"-version = "5.3.7"+version = "5.5.4"source = { registry = "https://pypi.org/simple" }dependencies = [{ name = "amqp" },+ { name = "packaging" },+ { name = "tzdata" },{ name = "vine" },]-sdist = { url = "https://files.pythonhosted.org/packages/99/3a/2fb09708fef243e35c286414f2bf78543dc311ae7d3de5d343bd8437e38d/kombu-5.3.7.tar.gz", hash = "sha256:011c4cd9a355c14a1de8d35d257314a1d2456d52b7140388561acac3cf1a97bf", size = 439344 }+sdist = { url = "https://files.pythonhosted.org/packages/0f/d3/5ff936d8319ac86b9c409f1501b07c426e6ad41966fedace9ef1b966e23f/kombu-5.5.4.tar.gz", hash = "sha256:886600168275ebeada93b888e831352fe578168342f0d1d5833d88ba0d847363", size = 461992 }wheels = [- { url = "https://files.pythonhosted.org/packages/b4/9a/1951f2261271d6994f0df5a55b3e9cdad42ed1fc3020a0dc7f6de80a4566/kombu-5.3.7-py3-none-any.whl", hash = "sha256:5634c511926309c7f9789f1433e9ed402616b56836ef9878f01bd59267b4c7a9", size = 200190 },+ { url = "https://files.pythonhosted.org/packages/ef/70/a07dcf4f62598c8ad579df241af55ced65bed76e42e45d3c368a6d82dbc1/kombu-5.5.4-py3-none-any.whl", hash = "sha256:a12ed0557c238897d8e518f1d1fdf84bd1516c5e305af2dacd85c2015115feb8", size = 210034 },][[package]]@@ -328,6 +381,15 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/70/8e/0e2d847013cb52cd35b38c009bb167a1a26b2ce6cd6965bf26b47bc0bf44/requests-2.31.0-py3-none-any.whl", hash = "sha256:58cd2187c01e70e6e26505bca751777aa9f2ee0b7f4300988b709f44e013003f", size = 62574 },]+[[package]]+name = "sniffio"+version = "1.3.1"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 },+]+[[package]]name = "typing-extensions"version = "4.13.2"@@ -337,11 +399,21 @@ wheels = [{ url = "https://files.pythonhosted.org/packages/8b/54/b1ae86c0973cc6f0210b53d508ca3641fb6d0c56823f288d108bc7ab3cc8/typing_extensions-4.13.2-py3-none-any.whl", hash = "sha256:a439e7c04b49fec3e5d3e2beaa21755cadbbdc391694e28ccdd36ca4a1408f8c", size = 45806 },]+[[package]]+name = "tzdata"+version = "2025.2"+source = { registry = "https://pypi.org/simple" }+sdist = { url = "https://files.pythonhosted.org/packages/95/32/1a225d6164441be760d75c2c42e2780dc0873fe382da3e98a2e1e48361e5/tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9", size = 196380 }+wheels = [+ { url = "https://files.pythonhosted.org/packages/5c/23/c7abc0ca0a1526a0774eca151daeb8de62ec457e77262b66b359c3c7679e/tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8", size = 347839 },+]+[[package]]name = "unstract-core"version = "0.0.1"source = { editable = "../unstract/core" }dependencies = [+ { name = "httpx" },{ name = "kombu" },{ name = "redis" },{ name = "requests" },@@ -350,7 +422,8 @@ dependencies = [[package.metadata]requires-dist = [{ name = "flask", marker = "extra == 'flask'", specifier = "~=3.1.0" },- { name = "kombu", specifier = "==5.3.7" },+ { name = "httpx", specifier = ">=0.27.0" },+ { name = "kombu", specifier = "~=5.5.3" },{ name = "redis", specifier = "~=5.2.1" },{ name = "requests", specifier = "==2.31.0" },]diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex c0a2d0715..966c925c2 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -82,8 +82,7 @@ class WorkflowExecutionService:logger.info(f"Execution {execution_id}: compiling workflow started")self.log_stage = LogStage.COMPILElog_message = (- f"Compiling workflow '{self.workflow_id}' "- f"of {len(self.tool_instances)} steps"+ f"Compiling workflow '{self.workflow_id}' of {len(self.tool_instances)} steps")self.publish_log(log_message)@@ -98,7 +97,7 @@ class WorkflowExecutionService:logger.info(f"Execution {execution_id}: compilation completed")log_message = (- f"Workflow '{self.workflow_id}' is valid " "and is compiled successfully"+ f"Workflow '{self.workflow_id}' is valid and is compiled successfully")self.publish_log(log_message)@@ -119,8 +118,7 @@ class WorkflowExecutionService:logger.info(f"Execution {self.execution_id}: Build started")self.log_stage = LogStage.BUILDlog_message = (- f"Building workflow '{self.workflow_id}' "- f"of {len(self.tool_instances)} steps"+ f"Building workflow '{self.workflow_id}' of {len(self.tool_instances)} steps")self.publish_log(log_message)@@ -130,8 +128,7 @@ class WorkflowExecutionService:)log_message = (- f"Workflow built successfully. Built tools = "- f"{len(self.tool_instances)}"+ f"Workflow built successfully. Built tools = {len(self.tool_instances)}")self.publish_log(log_message)except Exception as exception:@@ -193,8 +190,7 @@ class WorkflowExecutionService:tool_instance_id = sandbox.get_tool_instance_id()log_message = f"Executing step {actual_step} with tool {tool_uid}"logger.info(- f"Execution {self.execution_id}, Run {self.file_execution_id}"- f": {log_message}"+ f"Execution {self.execution_id}, Run {self.file_execution_id}: {log_message}")# TODO: Mention run_id in the FE logs / componentsself.publish_log(@@ -264,7 +260,7 @@ class WorkflowExecutionService:message="1",component=LogComponent.NEXT_STEP,)- log_message = f"Execution '{self.execution_id}' " "is waiting for user input"+ log_message = f"Execution '{self.execution_id}' is waiting for user input"self.publish_log(log_message)wait_for_user = 0@@ -277,9 +273,7 @@ class WorkflowExecutionService:if execution_value:execution_action = ExecutionAction(execution_value)if execution_action == ExecutionAction.NEXT:- log_message = (- f"Execution '{self.execution_id}' Executing " "NEXT step"- )+ log_message = f"Execution '{self.execution_id}' Executing NEXT step"self.publish_log(log_message)breakif execution_action == ExecutionAction.CONTINUE:@@ -291,7 +285,7 @@ class WorkflowExecutionService:self.override_single_step = Truebreakif execution_action == ExecutionAction.STOP:- log_message = f"Execution '{self.execution_id}' " "STOPPING execution"+ log_message = f"Execution '{self.execution_id}' STOPPING execution"self.publish_log(log_message)raise StopExecution("User clicked on stop button. Stopping execution")time.sleep(1)