LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
reverse-merge: MIME type fix for large files (for v0.106.2) (#1118)
Chandrasekharan M38a0d9ca6fbf
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex fe8e1dce6..15f37f911 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -9,6 +9,7 @@ import fsspecimport magicfrom connector_v2.models import ConnectorInstancefrom fsspec.implementations.local import LocalFileSystem+from rest_framework.exceptions import APIExceptionfrom unstract.sdk.constants import ToolExecKeyfrom unstract.sdk.file_storage.constants import FileOperationParamsfrom unstract.sdk.tool.mime_types import EXT_MIME_MAP@@ -426,25 +427,29 @@ class DestinationConnector(BaseConnector):result: Union[dict[str, Any], str] = ""try:# TODO: SDK handles validation; consider removing here.- mime = magic.Magic()- file_type = mime.from_file(output_file)+ with open(output_file, "rb") as f:+ file_type = magic.from_buffer(f.read(), mime=True)if output_type == ToolOutputType.JSON:- if "JSON" not in file_type:- logger.error(f"Output type json mismatched {file_type}")- raise ToolOutputTypeMismatch()+ if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:+ msg = f"Expected tool output type: JSON, got: '{file_type}'"+ logger.error(msg)+ raise ToolOutputTypeMismatch(detail=msg)with open(output_file) as file:result = json.load(file)elif output_type == ToolOutputType.TXT:- if "JSON" in file_type:- logger.error(f"Output type txt mismatched {file_type}")- raise ToolOutputTypeMismatch()+ if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:+ msg = f"Expected tool output type: TXT, got: '{file_type}'"+ logger.error(msg)+ raise ToolOutputTypeMismatch(detail=msg)with open(output_file) as file:result = file.read()result = result.encode("utf-8").decode("unicode-escape")else:raise InvalidToolOutputType()except (FileNotFoundError, json.JSONDecodeError) as err:- logger.error(f"Error while getting result {err}")+ msg = f"Error while getting result from the tool: {err}"+ logger.error(msg)+ raise APIException(detail=msg)return resultdef get_result_with_file_storage(@@ -470,20 +475,25 @@ class DestinationConnector(BaseConnector):)if output_type == ToolOutputType.JSON:if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:- logger.error(f"Output type json mismatched {file_type}")- raise ToolOutputTypeMismatch()+ msg = f"Expected tool output type: JSON, got: '{file_type}'"+ logger.error(msg)+ raise ToolOutputTypeMismatch(detail=msg)file_content = file_storage.read(output_file, mode="r")result = json.loads(file_content)elif output_type == ToolOutputType.TXT:if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:- logger.error(f"Output type txt mismatched {file_type}")- raise ToolOutputTypeMismatch()+ msg = f"Expected tool output type: TXT, got: '{file_type}'"+ logger.error(msg)+ raise ToolOutputTypeMismatch(detail=msg)file_content = file_storage.read(output_file, mode="r")result = file_content.encode("utf-8").decode("unicode-escape")else:raise InvalidToolOutputType()except (FileNotFoundError, json.JSONDecodeError) as err:- logger.error(f"Error while getting result {err}")+ msg = f"Error while getting result from the tool: {err}"+ logger.error(msg)+ raise APIException(detail=msg)+return resultdef get_metadata(diff --git a/backend/workflow_manager/endpoint_v2/exceptions.py b/backend/workflow_manager/endpoint_v2/exceptions.pyindex 9a1b03488..c8d644f8f 100644--- a/backend/workflow_manager/endpoint_v2/exceptions.py+++ b/backend/workflow_manager/endpoint_v2/exceptions.py@@ -73,14 +73,18 @@ class OrganizationIdNotFound(APIException):class InvalidToolOutputType(APIException):status_code = 500- default_detail = "Invalid output type is returned from tool"+ default_detail = "Unsupported output type is returned from tool"class ToolOutputTypeMismatch(APIException):- status_code = 400- default_detail = (- "The data type of the tool's output does not match the expected type."- )+ status_code = 500+ default_detail = "The tool's output type does not match the expected type"++ def __init__(self, detail: Optional[str] = None, code: Optional[str] = None):+ detail += (+ ". Please report this error to the administrator for further assistance."+ )+ super().__init__(detail, code)class BigQueryTableNotFound(APIException):diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.pyindex 5b1787690..bc7b795ab 100644--- a/backend/workflow_manager/workflow_v2/execution.py+++ b/backend/workflow_manager/workflow_v2/execution.py@@ -334,8 +334,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):if single_step:execution_type = ExecutionType.STEPself.publish_log(- "No entries found in cache, executing the tools"- f"running the tool(s) for {file_name}"+ f"No entries found in cache, executing the tool for '{file_name}'")self.publish_update_log(state=LogState.SUCCESS,diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.pyindex 4478ad359..095892474 100644--- a/backend/workflow_manager/workflow_v2/workflow_helper.py+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py@@ -149,7 +149,7 @@ class WorkflowHelper:if total_files > 0:q_file_no_list = WorkflowUtil.get_q_no_list(workflow, total_files)- for index, (file_path, file_hash) in enumerate(input_files.items()):+ for index, (file_name, file_hash) in enumerate(input_files.items()):# Get workflow execution fileworkflow_execution_file = cls._get_or_create_workflow_execution_file(execution_service=execution_service,@@ -194,7 +194,7 @@ class WorkflowHelper:breakexcept Exception as e:failed_files += 1- error_message = f"Error processing file '{file_path}'. {e}"+ error_message = f"Error processing file '{file_name}'. {e}"logger.error(error_message, stack_info=True, exc_info=True)workflow_execution_file.update_status(status=ExecutionStatus.ERROR,@@ -203,6 +203,7 @@ class WorkflowHelper:execution_service.publish_log(message=error_message, level=LogLevel.ERROR)+ # TODO: Review if we need partial successif failed_files and failed_files >= total_files:execution_service.update_execution(ExecutionStatus.ERROR, error=error_message