LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
handled never ending etl issue in MRQ when highlight is not selected (#570)
vishnuszipstack42ba8e0c4072
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.pyindex aafb5d624..5a65cc779 100644--- a/backend/workflow_manager/endpoint/destination.py+++ b/backend/workflow_manager/endpoint/destination.py@@ -528,17 +528,22 @@ class DestinationConnector(BaseConnector):settings=connector_settings, connector_id=connector.connector_id)with source_fs.open(input_file_path, "rb") as remote_file:+ whisper_hash = Nonefile_content = remote_file.read()# Convert file content to a base64 encoded stringfile_content_base64 = base64.b64encode(file_content).decode("utf-8")q_name = f"review_queue_{self.organization_id}_{workflow.id}"+ if meta_data:+ whisper_hash = meta_data.get("whisper-hash")+ else:+ whisper_hash = Nonequeue_result = QueueResult(file=file_name,- whisper_hash=meta_data["whisper-hash"],status=QueueResultStatus.SUCCESS,result=result,workflow_id=str(self.workflow_id),file_content=file_content_base64,+ whisper_hash=whisper_hash,).to_dict()# Convert the result dictionary to a JSON stringqueue_result_json = json.dumps(queue_result)diff --git a/backend/workflow_manager/endpoint/queue_utils.py b/backend/workflow_manager/endpoint/queue_utils.pyindex 17ca4e3c1..5311fadbf 100644--- a/backend/workflow_manager/endpoint/queue_utils.py+++ b/backend/workflow_manager/endpoint/queue_utils.py@@ -1,7 +1,7 @@import loggingfrom dataclasses import dataclassfrom enum import Enum-from typing import Any+from typing import Any, Optionalfrom utils.constants import Commonfrom workflow_manager.endpoint.exceptions import UnstractQueueException@@ -34,11 +34,11 @@ class QueueUtils:@dataclassclass QueueResult:file: str- whisper_hash: strstatus: QueueResultStatusresult: Anyworkflow_id: strfile_content: str+ whisper_hash: Optional[str] = Nonedef to_dict(self) -> Any:return {