LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
fix: File type support for WF and API deployments (#530)
Chandrasekharan M40b8af0de51a
diff --git a/backend/sample.env b/backend/sample.envindex 506f8644b..b7267ad22 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -90,9 +90,9 @@ PROMPT_STUDIO_FILE_PATH=/app/prompt-studio-data# Structure Tool Image (Runs prompt studio exported tools)# https://hub.docker.com/r/unstract/tool-structure-STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.33"+STRUCTURE_TOOL_IMAGE_URL="docker:unstract/tool-structure:0.0.34"STRUCTURE_TOOL_IMAGE_NAME="unstract/tool-structure"-STRUCTURE_TOOL_IMAGE_TAG="0.0.33"+STRUCTURE_TOOL_IMAGE_TAG="0.0.34"# Feature FlagsEVALUATION_SERVER_IP=unstract-fliptdiff --git a/backend/workflow_manager/endpoint/constants.py b/backend/workflow_manager/endpoint/constants.pyindex 01434b800..428d45ce5 100644--- a/backend/workflow_manager/endpoint/constants.py+++ b/backend/workflow_manager/endpoint/constants.py@@ -72,8 +72,8 @@ class FileType:class FilePattern:- PDF_DOCUMENTS = ["*.pdf", "*.PDF"]- TEXT_DOCUMENTS = ["*.txt", "*.TXT"]+ PDF_DOCUMENTS = ["*.pdf"]+ TEXT_DOCUMENTS = ["*.txt", "*.doc", "*.docx"]IMAGES = ["*.jpg","*.jpeg",@@ -82,13 +82,6 @@ class FilePattern:"*.bmp","*.tif","*.tiff",- "*.JPG",- "*.JPEG",- "*.PNG",- "*.GIF",- "*.BMP",- "*.TIF",- "*.TIFF",]diff --git a/backend/workflow_manager/endpoint/source.py b/backend/workflow_manager/endpoint/source.pyindex 33cb97484..d2707de0f 100644--- a/backend/workflow_manager/endpoint/source.py+++ b/backend/workflow_manager/endpoint/source.py@@ -251,7 +251,10 @@ class SourceConnector(BaseConnector):continueif count >= limit:break- if any(fnmatch.fnmatch(file, pattern) for pattern in patterns):+ if any(+ fnmatch.fnmatchcase(file.lower(), pattern.lower())+ for pattern in patterns+ ):file_path = os.path.join(root, file)file_path = f"{file_path}"matched_files.append(file_path)diff --git a/docker/scripts/merge_env.py b/docker/scripts/merge_env.pyindex 9a328b00c..73dcc4c6a 100644--- a/docker/scripts/merge_env.py+++ b/docker/scripts/merge_env.py@@ -10,10 +10,7 @@#import sys-PREFERRED_BASE_ENV_KEYS = [- "STRUCTURE_TOOL_IMAGE_URL",- "STRUCTURE_TOOL_IMAGE_TAG",-]+PREFERRED_BASE_ENV_KEYS = []DEFAULT_AUTH_KEY = "unstract"DEFAULT_ADMIN_KEY = "admin"SET_DEFAULT_KEYS = {diff --git a/prompt-service/src/unstract/prompt_service/main.py b/prompt-service/src/unstract/prompt_service/main.pyindex aeb9f022c..6a06b90e1 100644--- a/prompt-service/src/unstract/prompt_service/main.py+++ b/prompt-service/src/unstract/prompt_service/main.py@@ -178,7 +178,7 @@ def prompt_processor() -> Any:if not payload:raise NoPayloadErrortool_settings = payload.get(PSKeys.TOOL_SETTINGS, {})- outputs = payload.get(PSKeys.OUTPUTS)+ outputs = payload.get(PSKeys.OUTPUTS, [])tool_id: str = payload.get(PSKeys.TOOL_ID, "")run_id: str = payload.get(PSKeys.RUN_ID, "")file_hash = payload.get(PSKeys.FILE_HASH)@@ -198,7 +198,7 @@ def prompt_processor() -> Any:{"tool_id": tool_id, "run_id": run_id, "doc_name": doc_name},LogLevel.DEBUG,RunLevel.RUN,- "Preparing to execute all prompts",+ f"Preparing to execute {len(outputs)} prompt(s)",)for output in outputs: # type:ignorediff --git a/tools/structure/src/config/properties.json b/tools/structure/src/config/properties.jsonindex c45f18c62..8e7bb82ca 100644--- a/tools/structure/src/config/properties.json+++ b/tools/structure/src/config/properties.json@@ -2,7 +2,7 @@"schemaVersion": "0.0.1","displayName": "Structure Tool","functionName": "structure_tool",- "toolVersion": "0.0.33",+ "toolVersion": "0.0.34","description": "This is a template tool which can answer set of input prompts designed in the Prompt Studio","input": {"description": "File that needs to be indexed and parsed for answers"@@ -36,8 +36,7 @@"restrictions": {"maxFileSize": "50MB","allowedFileTypes": [- "txt",- "pdf"+ "*"]}}