LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
Handing API deployment errors (#507)
Rahul Johny535a2c28aa10
diff --git a/backend/sample.env b/backend/sample.envindex f605c45b6..cda1e835f 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -90,9 +90,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data# Structure Tool Image (Runs prompt studio exported tools)# https://hub.docker.com/r/unstract/tool-structure-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.31"+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.32"STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"-STRUCTURE_TOOL_IMAGE_TAG="0.0.31"+STRUCTURE_TOOL_IMAGE_TAG="0.0.32"# Feature FlagsEVALUATION_SERVER_IP=unstract-fliptdiff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txtindex 2a30fa417..225c3d2c0 100644--- a/tools/classifier/requirements.txt+++ b/tools/classifier/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.38.0+unstract-sdk~=0.38.1diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.jsonindex c86c089d9..a46b2c91c 100644--- a/tools/classifier/src/config/properties.json+++ b/tools/classifier/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "File Classifier","functionName": "classify",- "toolVersion": "0.0.27",+ "toolVersion": "0.0.28","description": "Classifies a file into a bin based on its contents","input": {"description": "File to be classified"diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txtindex 2a30fa417..225c3d2c0 100644--- a/tools/structure/requirements.txt+++ b/tools/structure/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.38.0+unstract-sdk~=0.38.1diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex 386d897d0..0e7400f0a 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.31",+ "toolVersion": "0.0.32","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txtindex 2a30fa417..225c3d2c0 100644--- a/tools/text_extractor/requirements.txt+++ b/tools/text_extractor/requirements.txt@@ -1,4 +1,4 @@# Add your dependencies here# Required for all unstract tools-unstract-sdk~=0.38.0+unstract-sdk~=0.38.1diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.jsonindex 5a85d1462..dcca4ca14 100644--- a/tools/text_extractor/src/config/properties.json+++ b/tools/text_extractor/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.25",+ "toolVersion": "0.0.26","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"diff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.jsonindex 729679652..e09e4aaed 100644--- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json+++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json@@ -5,7 +5,7 @@"schemaVersion": "0.0.1","displayName": "File Classifier","functionName": "classify",- "toolVersion": "0.0.27",+ "toolVersion": "0.0.28","description": "Classifies a file into a bin based on its contents","input": {"description": "File to be classified"@@ -139,9 +139,9 @@}},"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",- "image_url": "docker:unstract/tool-classifier:0.0.27",+ "image_url": "docker:unstract/tool-classifier:0.0.28","image_name": "unstract/tool-classifier",- "image_tag": "0.0.27"+ "image_tag": "0.0.28"},"text_extractor": {"tool_uid": "text_extractor",@@ -149,7 +149,7 @@"schemaVersion": "0.0.1","displayName": "Text Extractor","functionName": "text_extractor",- "toolVersion": "0.0.25",+ "toolVersion": "0.0.26","description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents","input": {"description": "Document"@@ -224,8 +224,8 @@}},"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",- "image_url": "docker:unstract/tool-text-extractor:0.0.25",+ "image_url": "docker:unstract/tool-text-extractor:0.0.26","image_name": "unstract/tool-text-extractor",- "image_tag": "0.0.25"+ "image_tag": "0.0.26"}}diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.pyindex b138c4089..d5fe76804 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py@@ -185,8 +185,7 @@ class ToolsUtils:try:response = tool_sandbox.run_tool()if response:- result: dict[str, Any] = response.get("result", {})- return result+ return responselogger.warning(f"ToolExecutionException - Retrying "f"({retry_count + 1}/{max_retries})"diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.pyindex 1f8bce13d..270ebb670 100644--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py@@ -201,9 +201,11 @@ class WorkflowExecutionService:message="Ready for execution",component=tool_instance_id,)- self.tool_utils.run_tool(+ result = self.tool_utils.run_tool(tool_sandbox=sandbox,)+ if result and result.get("error"):+ raise ToolOutputNotFoundException(result.get("error"))if not self.validate_execution_result(step + 1):raise ToolOutputNotFoundException(f"Tool exception raised for tool {tool_uid}, "diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.pyindex 9a6e02810..92c3a7b11 100644--- a/worker/src/unstract/worker/constants.py+++ b/worker/src/unstract/worker/constants.py@@ -6,6 +6,13 @@ class LogType:SINGLE_STEP = "SINGLE_STEP_MESSAGE"+class LogLevel:+ ERROR = "ERROR"+ WARN = "WARN"+ INFO = "INFO"+ DEBUG = "DEBUG"++class ToolKey:TOOL_INSTANCE_ID = "tool_instance_id"diff --git a/worker/src/unstract/worker/exception.py b/worker/src/unstract/worker/exception.pynew file mode 100644index 000000000..c7c639877--- /dev/null+++ b/worker/src/unstract/worker/exception.py@@ -0,0 +1,4 @@+class ToolRunException(Exception):+ def __init__(self, message):+ self.message = message+ super().__init__(self.message)diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.pyindex d62a058bd..30abbfe5a 100644--- a/worker/src/unstract/worker/worker.py+++ b/worker/src/unstract/worker/worker.py@@ -11,7 +11,8 @@ from unstract.worker.clients.interface import (ContainerClientInterface,ContainerInterface,)-from unstract.worker.constants import Env, LogType, ToolKey+from unstract.worker.constants import Env, LogLevel, LogType, ToolKey+from unstract.worker.exception import ToolRunExceptionfrom unstract.core.constants import LogFieldNamefrom unstract.core.pubsub_helper import LogPublisher@@ -79,6 +80,9 @@ class UnstractWorker:if not log_dict:return Nonelog_type = self.get_log_type(log_dict)+ log_level = self.get_log_level(log_dict)+ if log_type == LogType.LOG and log_level == LogLevel.ERROR:+ raise ToolRunException(log_dict.get("log"))if not self.is_valid_log_type(log_type):self.logger.warning(f"Received invalid logType: {log_type} with log message: {log_dict}"@@ -114,6 +118,12 @@ class UnstractWorker:return log_typereturn None+ def get_log_level(self, log_dict: Any) -> Optional[str]:+ if isinstance(log_dict, dict):+ log_level: Optional[str] = log_dict.get("level")+ return log_level+ return None+def run_command(self, command: str) -> Optional[Any]:"""Runs any given command on the container.@@ -210,6 +220,13 @@ class UnstractWorker:execution_id=execution_id,organization_id=organization_id,)+ except ToolRunException as te:+ self.logger.error(+ f"Error while running docker container: {te}",+ stack_info=True,+ exc_info=True,+ )+ result = {"type": "RESULT", "result": None, "error": str(te.message)}except Exception as e:self.logger.error(f"Failed to run docker container: {e}", stack_info=True, exc_info=True