Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

reverse-merge: MIME type fix for large files (for v0.106.2) (#1118)

Chandrasekharan M38a0d9ca6fbf
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index fe8e1dce6..15f37f911 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -9,6 +9,7 @@ import fsspec
import magic
from connector_v2.models import ConnectorInstance
from fsspec.implementations.local import LocalFileSystem
+from rest_framework.exceptions import APIException
from unstract.sdk.constants import ToolExecKey
from unstract.sdk.file_storage.constants import FileOperationParams
from unstract.sdk.tool.mime_types import EXT_MIME_MAP
@@ -426,25 +427,29 @@ class DestinationConnector(BaseConnector):
result: Union[dict[str, Any], str] = ""
try:
# TODO: SDK handles validation; consider removing here.
- mime = magic.Magic()
- file_type = mime.from_file(output_file)
+ with open(output_file, "rb") as f:
+ file_type = magic.from_buffer(f.read(), mime=True)
if output_type == ToolOutputType.JSON:
- if "JSON" not in file_type:
- logger.error(f"Output type json mismatched {file_type}")
- raise ToolOutputTypeMismatch()
+ if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
+ msg = f"Expected tool output type: JSON, got: '{file_type}'"
+ logger.error(msg)
+ raise ToolOutputTypeMismatch(detail=msg)
with open(output_file) as file:
result = json.load(file)
elif output_type == ToolOutputType.TXT:
- if "JSON" in file_type:
- logger.error(f"Output type txt mismatched {file_type}")
- raise ToolOutputTypeMismatch()
+ if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
+ msg = f"Expected tool output type: TXT, got: '{file_type}'"
+ logger.error(msg)
+ raise ToolOutputTypeMismatch(detail=msg)
with open(output_file) as file:
result = file.read()
result = result.encode("utf-8").decode("unicode-escape")
else:
raise InvalidToolOutputType()
except (FileNotFoundError, json.JSONDecodeError) as err:
- logger.error(f"Error while getting result {err}")
+ msg = f"Error while getting result from the tool: {err}"
+ logger.error(msg)
+ raise APIException(detail=msg)
return result
def get_result_with_file_storage(
@@ -470,20 +475,25 @@ class DestinationConnector(BaseConnector):
)
if output_type == ToolOutputType.JSON:
if file_type != EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
- logger.error(f"Output type json mismatched {file_type}")
- raise ToolOutputTypeMismatch()
+ msg = f"Expected tool output type: JSON, got: '{file_type}'"
+ logger.error(msg)
+ raise ToolOutputTypeMismatch(detail=msg)
file_content = file_storage.read(output_file, mode="r")
result = json.loads(file_content)
elif output_type == ToolOutputType.TXT:
if file_type == EXT_MIME_MAP[ToolOutputType.JSON.lower()]:
- logger.error(f"Output type txt mismatched {file_type}")
- raise ToolOutputTypeMismatch()
+ msg = f"Expected tool output type: TXT, got: '{file_type}'"
+ logger.error(msg)
+ raise ToolOutputTypeMismatch(detail=msg)
file_content = file_storage.read(output_file, mode="r")
result = file_content.encode("utf-8").decode("unicode-escape")
else:
raise InvalidToolOutputType()
except (FileNotFoundError, json.JSONDecodeError) as err:
- logger.error(f"Error while getting result {err}")
+ msg = f"Error while getting result from the tool: {err}"
+ logger.error(msg)
+ raise APIException(detail=msg)
+
return result
def get_metadata(
diff --git a/backend/workflow_manager/endpoint_v2/exceptions.py b/backend/workflow_manager/endpoint_v2/exceptions.py
index 9a1b03488..c8d644f8f 100644
--- a/backend/workflow_manager/endpoint_v2/exceptions.py
+++ b/backend/workflow_manager/endpoint_v2/exceptions.py
@@ -73,14 +73,18 @@ class OrganizationIdNotFound(APIException):
class InvalidToolOutputType(APIException):
status_code = 500
- default_detail = "Invalid output type is returned from tool"
+ default_detail = "Unsupported output type is returned from tool"
class ToolOutputTypeMismatch(APIException):
- status_code = 400
- default_detail = (
- "The data type of the tool's output does not match the expected type."
- )
+ status_code = 500
+ default_detail = "The tool's output type does not match the expected type"
+
+ def __init__(self, detail: Optional[str] = None, code: Optional[str] = None):
+ detail += (
+ ". Please report this error to the administrator for further assistance."
+ )
+ super().__init__(detail, code)
class BigQueryTableNotFound(APIException):
diff --git a/backend/workflow_manager/workflow_v2/execution.py b/backend/workflow_manager/workflow_v2/execution.py
index 5b1787690..bc7b795ab 100644
--- a/backend/workflow_manager/workflow_v2/execution.py
+++ b/backend/workflow_manager/workflow_v2/execution.py
@@ -334,8 +334,7 @@ class WorkflowExecutionServiceHelper(WorkflowExecutionService):
if single_step:
execution_type = ExecutionType.STEP
self.publish_log(
- "No entries found in cache, executing the tools"
- f"running the tool(s) for {file_name}"
+ f"No entries found in cache, executing the tool for '{file_name}'"
)
self.publish_update_log(
state=LogState.SUCCESS,
diff --git a/backend/workflow_manager/workflow_v2/workflow_helper.py b/backend/workflow_manager/workflow_v2/workflow_helper.py
index 4478ad359..095892474 100644
--- a/backend/workflow_manager/workflow_v2/workflow_helper.py
+++ b/backend/workflow_manager/workflow_v2/workflow_helper.py
@@ -149,7 +149,7 @@ class WorkflowHelper:
if total_files > 0:
q_file_no_list = WorkflowUtil.get_q_no_list(workflow, total_files)
- for index, (file_path, file_hash) in enumerate(input_files.items()):
+ for index, (file_name, file_hash) in enumerate(input_files.items()):
# Get workflow execution file
workflow_execution_file = cls._get_or_create_workflow_execution_file(
execution_service=execution_service,
@@ -194,7 +194,7 @@ class WorkflowHelper:
break
except Exception as e:
failed_files += 1
- error_message = f"Error processing file '{file_path}'. {e}"
+ error_message = f"Error processing file '{file_name}'. {e}"
logger.error(error_message, stack_info=True, exc_info=True)
workflow_execution_file.update_status(
status=ExecutionStatus.ERROR,
@@ -203,6 +203,7 @@ class WorkflowHelper:
execution_service.publish_log(
message=error_message, level=LogLevel.ERROR
)
+ # TODO: Review if we need partial success
if failed_files and failed_files >= total_files:
execution_service.update_execution(
ExecutionStatus.ERROR, error=error_message