LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
UN-2417 [FEAT] HITL TTL (#1441)
vishnuszipstack6f23ce9aca56
diff --git a/backend/plugins/workflow_manager/workflow_v2/utils.py b/backend/plugins/workflow_manager/workflow_v2/utils.pyindex 14917afcb..b96034ec6 100644--- a/backend/plugins/workflow_manager/workflow_v2/utils.py+++ b/backend/plugins/workflow_manager/workflow_v2/utils.py@@ -81,3 +81,17 @@ class WorkflowUtil:and file destination."""return False++ @staticmethod+ def get_hitl_ttl_seconds(workflow: Workflow) -> Any:+ """Get TTL in seconds for HITL settings for a workflow.++ Args:+ workflow (Workflow): The workflow to get HITL TTL settings for.++ Returns:+ Optional[int]: TTL in seconds if custom TTL is set, None for unlimited TTL.+ For OSS version, always returns None (unlimited TTL).+ """+ # TTL feature is cloud-only, return None for unlimited TTL in OSS+ return Nonediff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.pyindex ab3c7fd4f..c5a3cb890 100644--- a/backend/workflow_manager/endpoint_v2/destination.py+++ b/backend/workflow_manager/endpoint_v2/destination.py@@ -789,6 +789,7 @@ class DestinationConnector(BaseConnector):).to_dict()queue_result_json = json.dumps(queue_result)+conn = QueueUtils.get_queue_inst()conn.enqueue(queue_name=q_name, message=queue_result_json)logger.info(f"Pushed {file_name} to queue {q_name} with file content")@@ -811,7 +812,9 @@ class DestinationConnector(BaseConnector):whisper_hash = meta_data.get("whisper-hash")else:whisper_hash = None- queue_result = QueueResult(++ # Create QueueResult with TTL metadata+ queue_result_obj = QueueResult(file=file_name,status=QueueResultStatus.SUCCESS,result=result,@@ -819,38 +822,27 @@ class DestinationConnector(BaseConnector):file_content=file_content_base64,whisper_hash=whisper_hash,file_execution_id=file_execution_id,- ).to_dict()- # Convert the result dictionary to a JSON string- queue_result_json = json.dumps(queue_result)- conn = QueueUtils.get_queue_inst()- # Enqueue the JSON string- conn.enqueue(queue_name=q_name, message=queue_result_json)- logger.info(f"Pushed {file_name} to queue {q_name} with file content")+ )- def _read_file_content_for_queue(self, input_file_path: str, file_name: str) -> str:- """Read and encode file content for queue message.+ # Add TTL metadata based on HITLSettings+ queue_result_obj.ttl_seconds = WorkflowUtil.get_hitl_ttl_seconds(workflow)- Args:- input_file_path: Path to the file to read- file_name: Name of the file for logging purposes+ queue_result = queue_result_obj.to_dict()+ queue_result_json = json.dumps(queue_result)- Returns:- Base64 encoded file content+ # Validate the JSON is not empty before enqueuing+ if not queue_result_json or queue_result_json.strip() == "":+ logger.error(+ f"Attempted to enqueue empty JSON with TTL for file {file_name}"+ )+ raise ValueError("Cannot enqueue empty JSON message")- Raises:- APIException: If file cannot be read or doesn't exist- """- try:- file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)- file_storage = file_system.get_file_storage()+ conn = QueueUtils.get_queue_inst()- if not file_storage.exists(input_file_path):- raise APIException(f"File not found: {input_file_path}")+ # Use the TTL metadata that was already set in the QueueResult object+ ttl_seconds = queue_result_obj.ttl_seconds- file_bytes = file_storage.read(input_file_path, mode="rb")- if isinstance(file_bytes, str):- file_bytes = file_bytes.encode("utf-8")- return base64.b64encode(file_bytes).decode("utf-8")- except Exception as e:- logger.error(f"Failed to read file content for {file_name}: {e}")- raise APIException(f"Failed to read file content for queue: {e}")+ conn.enqueue_with_ttl(+ queue_name=q_name, message=queue_result_json, ttl_seconds=ttl_seconds+ )+ logger.info(f"Pushed {file_name} to queue {q_name} with file content")diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.pyindex f22e37300..d29660762 100644--- a/backend/workflow_manager/endpoint_v2/queue_utils.py+++ b/backend/workflow_manager/endpoint_v2/queue_utils.py@@ -1,4 +1,5 @@import logging+import timefrom dataclasses import dataclassfrom enum import Enumfrom typing import Any@@ -30,6 +31,60 @@ class QueueUtils:connector_class: UnstractQueue = connector(connector_settings)return connector_class+ @staticmethod+ def calculate_remaining_ttl(enqueued_at: float, ttl_seconds: int) -> int | None:+ """Calculate remaining TTL based on original enqueue time and TTL.++ Args:+ enqueued_at: Timestamp when record was first enqueued+ ttl_seconds: TTL in seconds++ Returns:+ Remaining TTL in seconds, or None if expired+ """+ if ttl_seconds is None:+ return None # Unlimited TTL++ current_time = time.time()+ elapsed_time = current_time - enqueued_at+ remaining_ttl = ttl_seconds - int(elapsed_time)++ return max(0, remaining_ttl) if remaining_ttl > 0 else None++ @staticmethod+ def is_ttl_expired(enqueued_at: float, ttl_seconds: int) -> bool:+ """Check if TTL has expired for a record.++ Args:+ enqueued_at: Timestamp when record was first enqueued+ ttl_seconds: TTL in seconds++ Returns:+ True if TTL has expired, False otherwise+ """+ if ttl_seconds is None:+ return False # Unlimited TTL never expires++ remaining_ttl = QueueUtils.calculate_remaining_ttl(enqueued_at, ttl_seconds)+ return remaining_ttl is None or remaining_ttl <= 0++ @staticmethod+ def create_queue_result_with_ttl(+ queue_result: "QueueResult", ttl_seconds: int | None = None+ ) -> "QueueResult":+ """Create a QueueResult with TTL metadata.++ Args:+ queue_result: Original QueueResult+ ttl_seconds: TTL in seconds++ Returns:+ QueueResult with TTL metadata+ """+ if queue_result.ttl_seconds is None and ttl_seconds is not None:+ queue_result.ttl_seconds = ttl_seconds+ return queue_result+@dataclassclass QueueResult:@@ -40,9 +95,24 @@ class QueueResult:file_content: strwhisper_hash: str | None = Nonefile_execution_id: str | None = None+ enqueued_at: float | None = None+ ttl_seconds: int | None = None++ def __post_init__(self):+ """Initialize enqueued_at timestamp if not provided and validate required fields"""+ if self.enqueued_at is None:+ self.enqueued_at = time.time()++ # Validate required fields+ if not self.file:+ raise ValueError("QueueResult requires a valid file name")+ if not self.workflow_id:+ raise ValueError("QueueResult requires a valid workflow_id")+ if self.status is None:+ raise ValueError("QueueResult requires a valid status")def to_dict(self) -> Any:- return {+ result_dict = {"file": self.file,"whisper_hash": self.whisper_hash,"status": self.status.value,@@ -50,4 +120,7 @@ class QueueResult:"workflow_id": self.workflow_id,"file_content": self.file_content,"file_execution_id": self.file_execution_id,+ "enqueued_at": self.enqueued_at,+ "ttl_seconds": self.ttl_seconds,}+ return result_dictdiff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.jsindex af6e7a19d..fc00b28ca 100644--- a/frontend/src/helpers/GetStaticData.js+++ b/frontend/src/helpers/GetStaticData.js@@ -648,6 +648,22 @@ const convertTimestampToHHMMSS = (timestamp) => {return `${hours}:${minutes}:${seconds}`;};+const formatTimeDisplay = (seconds) => {+ // Format time display for TTL or duration display+ if (seconds <= 0) return "Expired";++ const hours = Math.floor(seconds / 3600);+ const minutes = Math.floor((seconds % 3600) / 60);+ const secs = Math.floor(seconds % 60);++ const parts = [];+ if (hours > 0) parts.push(`${hours}h`);+ if (minutes > 0) parts.push(`${minutes}m`);+ if (secs > 0 || parts.length === 0) parts.push(`${secs}s`);++ return parts.join(" ");+};+const UNSTRACT_ADMIN = "unstract_admin";const logsStaticContent = {@@ -725,6 +741,7 @@ export {convertTimestampToHHMMSS,UNSTRACT_ADMIN,formatSecondsToHMS,+ formatTimeDisplay,formattedDateTimeWithSeconds,logsStaticContent,};diff --git a/frontend/src/hooks/useExceptionHandler.jsx b/frontend/src/hooks/useExceptionHandler.jsxindex 226bed3db..dc458d88f 100644--- a/frontend/src/hooks/useExceptionHandler.jsx+++ b/frontend/src/hooks/useExceptionHandler.jsx@@ -35,7 +35,23 @@ const useExceptionHandler = () => {}if (err?.response?.data) {- const { type, errors } = err.response.data;+ const responseData = err.response.data;+ const { type, errors } = responseData;++ // First, try to extract common API error messages (for DRF and other APIs)+ const commonErrorMessage =+ responseData.error || responseData.detail || responseData.message;++ if (commonErrorMessage) {+ return {+ title: title,+ type: "error",+ content: commonErrorMessage,+ duration: duration,+ };+ }++ // Then handle specific error typesswitch (type) {case "validation_error":// Handle validation errors