Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

Handing API deployment errors (#507)

Rahul Johny535a2c28aa10
diff --git a/backend/sample.env b/backend/sample.env
index f605c45b6..cda1e835f 100644
--- a/backend/sample.env
+++ b/backend/sample.env
@@ -90,9 +90,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data
# Structure Tool Image (Runs prompt studio exported tools)
# https://hub.docker.com/r/unstract/tool-structure
-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.31"
+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.32"
STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"
-STRUCTURE_TOOL_IMAGE_TAG="0.0.31"
+STRUCTURE_TOOL_IMAGE_TAG="0.0.32"
# Feature Flags
EVALUATION_SERVER_IP=unstract-flipt
diff --git a/tools/classifier/requirements.txt b/tools/classifier/requirements.txt
index 2a30fa417..225c3d2c0 100644
--- a/tools/classifier/requirements.txt
+++ b/tools/classifier/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.38.0
+unstract-sdk~=0.38.1
diff --git a/tools/classifier/src/config/properties.json b/tools/classifier/src/config/properties.json
index c86c089d9..a46b2c91c 100644
--- a/tools/classifier/src/config/properties.json
+++ b/tools/classifier/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.27",
+ "toolVersion": "0.0.28",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
diff --git a/tools/structure/requirements.txt b/tools/structure/requirements.txt
index 2a30fa417..225c3d2c0 100644
--- a/tools/structure/requirements.txt
+++ b/tools/structure/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.38.0
+unstract-sdk~=0.38.1
diff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.json
index 386d897d0..0e7400f0a 100644
--- a/tools/structure/src/config/properties.json
+++ b/tools/structure/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Structure Tool",
"functionName": "structure_tool",
- "toolVersion": "0.0.31",
+ "toolVersion": "0.0.32",
"description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio",
"input": {
"description": "File that needs to be indexed and parsed for answers"
diff --git a/tools/text_extractor/requirements.txt b/tools/text_extractor/requirements.txt
index 2a30fa417..225c3d2c0 100644
--- a/tools/text_extractor/requirements.txt
+++ b/tools/text_extractor/requirements.txt
@@ -1,4 +1,4 @@
# Add your dependencies here
# Required for all unstract tools
-unstract-sdk~=0.38.0
+unstract-sdk~=0.38.1
diff --git a/tools/text_extractor/src/config/properties.json b/tools/text_extractor/src/config/properties.json
index 5a85d1462..dcca4ca14 100644
--- a/tools/text_extractor/src/config/properties.json
+++ b/tools/text_extractor/src/config/properties.json
@@ -2,7 +2,7 @@
"schemaVersion": "0.0.1",
"displayName": "Text Extractor",
"functionName": "text_extractor",
- "toolVersion": "0.0.25",
+ "toolVersion": "0.0.26",
"description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents",
"input": {
"description": "Document"
diff --git a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
index 729679652..e09e4aaed 100644
--- a/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
+++ b/unstract/tool-registry/src/unstract/tool_registry/public_tools.json
@@ -5,7 +5,7 @@
"schemaVersion": "0.0.1",
"displayName": "File Classifier",
"functionName": "classify",
- "toolVersion": "0.0.27",
+ "toolVersion": "0.0.28",
"description": "Classifies a file into a bin based on its contents",
"input": {
"description": "File to be classified"
@@ -139,9 +139,9 @@
}
},
"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",
- "image_url": "docker:unstract/tool-classifier:0.0.27",
+ "image_url": "docker:unstract/tool-classifier:0.0.28",
"image_name": "unstract/tool-classifier",
- "image_tag": "0.0.27"
+ "image_tag": "0.0.28"
},
"text_extractor": {
"tool_uid": "text_extractor",
@@ -149,7 +149,7 @@
"schemaVersion": "0.0.1",
"displayName": "Text Extractor",
"functionName": "text_extractor",
- "toolVersion": "0.0.25",
+ "toolVersion": "0.0.26",
"description": "The Text Extractor is a powerful tool designed to convert documents to its text form or Extract texts from documents",
"input": {
"description": "Document"
@@ -224,8 +224,8 @@
}
},
"icon": "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n<svg\n enable-background=\"new 0 0 20 20\"\n height=\"48\"\n viewBox=\"0 0 20 20\"\n width=\"48\"\n fill=\"#000000\"\n version=\"1.1\"\n id=\"svg8109\"\n sodipodi:docname=\"folder_copy_black_48dp.svg\"\n xmlns:inkscape=\"http://www.inkscape.org/namespaces/inkscape\"\n xmlns:sodipodi=\"http://sodipodi.sourceforge.net/DTD/sodipodi-0.dtd\"\n xmlns=\"http://www.w3.org/2000/svg\"\n xmlns:svg=\"http://www.w3.org/2000/svg\">\n <defs\n id=\"defs8113\" />\n <sodipodi:namedview\n id=\"namedview8111\"\n pagecolor=\"#ffffff\"\n bordercolor=\"#000000\"\n borderopacity=\"0.25\"\n inkscape:showpageshadow=\"2\"\n inkscape:pageopacity=\"0.0\"\n inkscape:pagecheckerboard=\"0\"\n inkscape:deskcolor=\"#d1d1d1\"\n showgrid=\"false\" />\n <g\n id=\"g8099\">\n <rect\n fill=\"none\"\n height=\"20\"\n width=\"20\"\n x=\"0\"\n id=\"rect8097\"\n y=\"0\" />\n </g>\n <g\n id=\"g8107\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <g\n id=\"g8105\"\n style=\"fill:#ff4d6d;fill-opacity:1\">\n <path\n d=\"M 2.5,5 H 1 V 15.5 C 1,16.33 1.67,17 2.5,17 H 15.68 V 15.5 H 2.5 Z\"\n id=\"path8101\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n <path\n d=\"M 16.5,4 H 11 L 9,2 H 5.5 C 4.67,2 4,2.67 4,3.5 v 9 C 4,13.33 4.67,14 5.5,14 h 11 c 0.83,0 1.5,-0.67 1.5,-1.5 v -7 C 18,4.67 17.33,4 16.5,4 Z m 0,8.5 h -11 v -9 h 2.88 l 2,2 h 6.12 z\"\n id=\"path8103\"\n style=\"fill:#ff4d6d;fill-opacity:1\" />\n </g>\n </g>\n</svg>\n",
- "image_url": "docker:unstract/tool-text-extractor:0.0.25",
+ "image_url": "docker:unstract/tool-text-extractor:0.0.26",
"image_name": "unstract/tool-text-extractor",
- "image_tag": "0.0.25"
+ "image_tag": "0.0.26"
}
}
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
index b138c4089..d5fe76804 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/tools_utils.py
@@ -185,8 +185,7 @@ class ToolsUtils:
try:
response = tool_sandbox.run_tool()
if response:
- result: dict[str, Any] = response.get("result", {})
- return result
+ return response
logger.warning(
f"ToolExecutionException - Retrying "
f"({retry_count + 1}/{max_retries})"
diff --git a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
index 1f8bce13d..270ebb670 100644
--- a/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
+++ b/unstract/workflow-execution/src/unstract/workflow_execution/workflow_execution.py
@@ -201,9 +201,11 @@ class WorkflowExecutionService:
message="Ready for execution",
component=tool_instance_id,
)
- self.tool_utils.run_tool(
+ result = self.tool_utils.run_tool(
tool_sandbox=sandbox,
)
+ if result and result.get("error"):
+ raise ToolOutputNotFoundException(result.get("error"))
if not self.validate_execution_result(step + 1):
raise ToolOutputNotFoundException(
f"Tool exception raised for tool {tool_uid}, "
diff --git a/worker/src/unstract/worker/constants.py b/worker/src/unstract/worker/constants.py
index 9a6e02810..92c3a7b11 100644
--- a/worker/src/unstract/worker/constants.py
+++ b/worker/src/unstract/worker/constants.py
@@ -6,6 +6,13 @@ class LogType:
SINGLE_STEP = "SINGLE_STEP_MESSAGE"
+class LogLevel:
+ ERROR = "ERROR"
+ WARN = "WARN"
+ INFO = "INFO"
+ DEBUG = "DEBUG"
+
+
class ToolKey:
TOOL_INSTANCE_ID = "tool_instance_id"
diff --git a/worker/src/unstract/worker/exception.py b/worker/src/unstract/worker/exception.py
new file mode 100644
index 000000000..c7c639877
--- /dev/null
+++ b/worker/src/unstract/worker/exception.py
@@ -0,0 +1,4 @@
+class ToolRunException(Exception):
+ def __init__(self, message):
+ self.message = message
+ super().__init__(self.message)
diff --git a/worker/src/unstract/worker/worker.py b/worker/src/unstract/worker/worker.py
index d62a058bd..30abbfe5a 100644
--- a/worker/src/unstract/worker/worker.py
+++ b/worker/src/unstract/worker/worker.py
@@ -11,7 +11,8 @@ from unstract.worker.clients.interface import (
ContainerClientInterface,
ContainerInterface,
)
-from unstract.worker.constants import Env, LogType, ToolKey
+from unstract.worker.constants import Env, LogLevel, LogType, ToolKey
+from unstract.worker.exception import ToolRunException
from unstract.core.constants import LogFieldName
from unstract.core.pubsub_helper import LogPublisher
@@ -79,6 +80,9 @@ class UnstractWorker:
if not log_dict:
return None
log_type = self.get_log_type(log_dict)
+ log_level = self.get_log_level(log_dict)
+ if log_type == LogType.LOG and log_level == LogLevel.ERROR:
+ raise ToolRunException(log_dict.get("log"))
if not self.is_valid_log_type(log_type):
self.logger.warning(
f"Received invalid logType: {log_type} with log message: {log_dict}"
@@ -114,6 +118,12 @@ class UnstractWorker:
return log_type
return None
+ def get_log_level(self, log_dict: Any) -> Optional[str]:
+ if isinstance(log_dict, dict):
+ log_level: Optional[str] = log_dict.get("level")
+ return log_level
+ return None
+
def run_command(self, command: str) -> Optional[Any]:
"""Runs any given command on the container.
@@ -210,6 +220,13 @@ class UnstractWorker:
execution_id=execution_id,
organization_id=organization_id,
)
+ except ToolRunException as te:
+ self.logger.error(
+ f"Error while running docker container: {te}",
+ stack_info=True,
+ exc_info=True,
+ )
+ result = {"type": "RESULT", "result": None, "error": str(te.message)}
except Exception as e:
self.logger.error(
f"Failed to run docker container: {e}", stack_info=True, exc_info=True