Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

handled never ending etl issue in MRQ when highlight is not selected (#570)

vishnuszipstack42ba8e0c4072
diff --git a/backend/workflow_manager/endpoint/destination.py b/backend/workflow_manager/endpoint/destination.py
index aafb5d624..5a65cc779 100644
--- a/backend/workflow_manager/endpoint/destination.py
+++ b/backend/workflow_manager/endpoint/destination.py
@@ -528,17 +528,22 @@ class DestinationConnector(BaseConnector):
settings=connector_settings, connector_id=connector.connector_id
)
with source_fs.open(input_file_path, "rb") as remote_file:
+ whisper_hash = None
file_content = remote_file.read()
# Convert file content to a base64 encoded string
file_content_base64 = base64.b64encode(file_content).decode("utf-8")
q_name = f"review_queue_{self.organization_id}_{workflow.id}"
+ if meta_data:
+ whisper_hash = meta_data.get("whisper-hash")
+ else:
+ whisper_hash = None
queue_result = QueueResult(
file=file_name,
- whisper_hash=meta_data["whisper-hash"],
status=QueueResultStatus.SUCCESS,
result=result,
workflow_id=str(self.workflow_id),
file_content=file_content_base64,
+ whisper_hash=whisper_hash,
).to_dict()
# Convert the result dictionary to a JSON string
queue_result_json = json.dumps(queue_result)
diff --git a/backend/workflow_manager/endpoint/queue_utils.py b/backend/workflow_manager/endpoint/queue_utils.py
index 17ca4e3c1..5311fadbf 100644
--- a/backend/workflow_manager/endpoint/queue_utils.py
+++ b/backend/workflow_manager/endpoint/queue_utils.py
@@ -1,7 +1,7 @@
import logging
from dataclasses import dataclass
from enum import Enum
-from typing import Any
+from typing import Any, Optional
from utils.constants import Common
from workflow_manager.endpoint.exceptions import UnstractQueueException
@@ -34,11 +34,11 @@ class QueueUtils:
@dataclass
class QueueResult:
file: str
- whisper_hash: str
status: QueueResultStatus
result: Any
workflow_id: str
file_content: str
+ whisper_hash: Optional[str] = None
def to_dict(self) -> Any:
return {