Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2417 [FEAT] HITL TTL (#1441)

vishnuszipstack6f23ce9aca56
diff --git a/backend/plugins/workflow_manager/workflow_v2/utils.py b/backend/plugins/workflow_manager/workflow_v2/utils.py
index 14917afcb..b96034ec6 100644
--- a/backend/plugins/workflow_manager/workflow_v2/utils.py
+++ b/backend/plugins/workflow_manager/workflow_v2/utils.py
@@ -81,3 +81,17 @@ class WorkflowUtil:
and file destination.
"""
return False
+
+ @staticmethod
+ def get_hitl_ttl_seconds(workflow: Workflow) -> Any:
+ """Get TTL in seconds for HITL settings for a workflow.
+
+ Args:
+ workflow (Workflow): The workflow to get HITL TTL settings for.
+
+ Returns:
+ Optional[int]: TTL in seconds if custom TTL is set, None for unlimited TTL.
+ For OSS version, always returns None (unlimited TTL).
+ """
+ # TTL feature is cloud-only, return None for unlimited TTL in OSS
+ return None
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index ab3c7fd4f..c5a3cb890 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -789,6 +789,7 @@ class DestinationConnector(BaseConnector):
).to_dict()
queue_result_json = json.dumps(queue_result)
+
conn = QueueUtils.get_queue_inst()
conn.enqueue(queue_name=q_name, message=queue_result_json)
logger.info(f"Pushed {file_name} to queue {q_name} with file content")
@@ -811,7 +812,9 @@ class DestinationConnector(BaseConnector):
whisper_hash = meta_data.get("whisper-hash")
else:
whisper_hash = None
- queue_result = QueueResult(
+
+ # Create QueueResult with TTL metadata
+ queue_result_obj = QueueResult(
file=file_name,
status=QueueResultStatus.SUCCESS,
result=result,
@@ -819,38 +822,27 @@ class DestinationConnector(BaseConnector):
file_content=file_content_base64,
whisper_hash=whisper_hash,
file_execution_id=file_execution_id,
- ).to_dict()
- # Convert the result dictionary to a JSON string
- queue_result_json = json.dumps(queue_result)
- conn = QueueUtils.get_queue_inst()
- # Enqueue the JSON string
- conn.enqueue(queue_name=q_name, message=queue_result_json)
- logger.info(f"Pushed {file_name} to queue {q_name} with file content")
+ )
- def _read_file_content_for_queue(self, input_file_path: str, file_name: str) -> str:
- """Read and encode file content for queue message.
+ # Add TTL metadata based on HITLSettings
+ queue_result_obj.ttl_seconds = WorkflowUtil.get_hitl_ttl_seconds(workflow)
- Args:
- input_file_path: Path to the file to read
- file_name: Name of the file for logging purposes
+ queue_result = queue_result_obj.to_dict()
+ queue_result_json = json.dumps(queue_result)
- Returns:
- Base64 encoded file content
+ # Validate the JSON is not empty before enqueuing
+ if not queue_result_json or queue_result_json.strip() == "":
+ logger.error(
+ f"Attempted to enqueue empty JSON with TTL for file {file_name}"
+ )
+ raise ValueError("Cannot enqueue empty JSON message")
- Raises:
- APIException: If file cannot be read or doesn't exist
- """
- try:
- file_system = FileSystem(FileStorageType.WORKFLOW_EXECUTION)
- file_storage = file_system.get_file_storage()
+ conn = QueueUtils.get_queue_inst()
- if not file_storage.exists(input_file_path):
- raise APIException(f"File not found: {input_file_path}")
+ # Use the TTL metadata that was already set in the QueueResult object
+ ttl_seconds = queue_result_obj.ttl_seconds
- file_bytes = file_storage.read(input_file_path, mode="rb")
- if isinstance(file_bytes, str):
- file_bytes = file_bytes.encode("utf-8")
- return base64.b64encode(file_bytes).decode("utf-8")
- except Exception as e:
- logger.error(f"Failed to read file content for {file_name}: {e}")
- raise APIException(f"Failed to read file content for queue: {e}")
+ conn.enqueue_with_ttl(
+ queue_name=q_name, message=queue_result_json, ttl_seconds=ttl_seconds
+ )
+ logger.info(f"Pushed {file_name} to queue {q_name} with file content")
diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.py
index f22e37300..d29660762 100644
--- a/backend/workflow_manager/endpoint_v2/queue_utils.py
+++ b/backend/workflow_manager/endpoint_v2/queue_utils.py
@@ -1,4 +1,5 @@
import logging
+import time
from dataclasses import dataclass
from enum import Enum
from typing import Any
@@ -30,6 +31,60 @@ class QueueUtils:
connector_class: UnstractQueue = connector(connector_settings)
return connector_class
+ @staticmethod
+ def calculate_remaining_ttl(enqueued_at: float, ttl_seconds: int) -> int | None:
+ """Calculate remaining TTL based on original enqueue time and TTL.
+
+ Args:
+ enqueued_at: Timestamp when record was first enqueued
+ ttl_seconds: TTL in seconds
+
+ Returns:
+ Remaining TTL in seconds, or None if expired
+ """
+ if ttl_seconds is None:
+ return None # Unlimited TTL
+
+ current_time = time.time()
+ elapsed_time = current_time - enqueued_at
+ remaining_ttl = ttl_seconds - int(elapsed_time)
+
+ return max(0, remaining_ttl) if remaining_ttl > 0 else None
+
+ @staticmethod
+ def is_ttl_expired(enqueued_at: float, ttl_seconds: int) -> bool:
+ """Check if TTL has expired for a record.
+
+ Args:
+ enqueued_at: Timestamp when record was first enqueued
+ ttl_seconds: TTL in seconds
+
+ Returns:
+ True if TTL has expired, False otherwise
+ """
+ if ttl_seconds is None:
+ return False # Unlimited TTL never expires
+
+ remaining_ttl = QueueUtils.calculate_remaining_ttl(enqueued_at, ttl_seconds)
+ return remaining_ttl is None or remaining_ttl <= 0
+
+ @staticmethod
+ def create_queue_result_with_ttl(
+ queue_result: "QueueResult", ttl_seconds: int | None = None
+ ) -> "QueueResult":
+ """Create a QueueResult with TTL metadata.
+
+ Args:
+ queue_result: Original QueueResult
+ ttl_seconds: TTL in seconds
+
+ Returns:
+ QueueResult with TTL metadata
+ """
+ if queue_result.ttl_seconds is None and ttl_seconds is not None:
+ queue_result.ttl_seconds = ttl_seconds
+ return queue_result
+
@dataclass
class QueueResult:
@@ -40,9 +95,24 @@ class QueueResult:
file_content: str
whisper_hash: str | None = None
file_execution_id: str | None = None
+ enqueued_at: float | None = None
+ ttl_seconds: int | None = None
+
+ def __post_init__(self):
+ """Initialize enqueued_at timestamp if not provided and validate required fields"""
+ if self.enqueued_at is None:
+ self.enqueued_at = time.time()
+
+ # Validate required fields
+ if not self.file:
+ raise ValueError("QueueResult requires a valid file name")
+ if not self.workflow_id:
+ raise ValueError("QueueResult requires a valid workflow_id")
+ if self.status is None:
+ raise ValueError("QueueResult requires a valid status")
def to_dict(self) -> Any:
- return {
+ result_dict = {
"file": self.file,
"whisper_hash": self.whisper_hash,
"status": self.status.value,
@@ -50,4 +120,7 @@ class QueueResult:
"workflow_id": self.workflow_id,
"file_content": self.file_content,
"file_execution_id": self.file_execution_id,
+ "enqueued_at": self.enqueued_at,
+ "ttl_seconds": self.ttl_seconds,
}
+ return result_dict
diff --git a/frontend/src/helpers/GetStaticData.js b/frontend/src/helpers/GetStaticData.js
index af6e7a19d..fc00b28ca 100644
--- a/frontend/src/helpers/GetStaticData.js
+++ b/frontend/src/helpers/GetStaticData.js
@@ -648,6 +648,22 @@ const convertTimestampToHHMMSS = (timestamp) => {
return `${hours}:${minutes}:${seconds}`;
};
+const formatTimeDisplay = (seconds) => {
+ // Format time display for TTL or duration display
+ if (seconds <= 0) return "Expired";
+
+ const hours = Math.floor(seconds / 3600);
+ const minutes = Math.floor((seconds % 3600) / 60);
+ const secs = Math.floor(seconds % 60);
+
+ const parts = [];
+ if (hours > 0) parts.push(`${hours}h`);
+ if (minutes > 0) parts.push(`${minutes}m`);
+ if (secs > 0 || parts.length === 0) parts.push(`${secs}s`);
+
+ return parts.join(" ");
+};
+
const UNSTRACT_ADMIN = "unstract_admin";
const logsStaticContent = {
@@ -725,6 +741,7 @@ export {
convertTimestampToHHMMSS,
UNSTRACT_ADMIN,
formatSecondsToHMS,
+ formatTimeDisplay,
formattedDateTimeWithSeconds,
logsStaticContent,
};
diff --git a/frontend/src/hooks/useExceptionHandler.jsx b/frontend/src/hooks/useExceptionHandler.jsx
index 226bed3db..dc458d88f 100644
--- a/frontend/src/hooks/useExceptionHandler.jsx
+++ b/frontend/src/hooks/useExceptionHandler.jsx
@@ -35,7 +35,23 @@ const useExceptionHandler = () => {
}
if (err?.response?.data) {
- const { type, errors } = err.response.data;
+ const responseData = err.response.data;
+ const { type, errors } = responseData;
+
+ // First, try to extract common API error messages (for DRF and other APIs)
+ const commonErrorMessage =
+ responseData.error || responseData.detail || responseData.message;
+
+ if (commonErrorMessage) {
+ return {
+ title: title,
+ type: "error",
+ content: commonErrorMessage,
+ duration: duration,
+ };
+ }
+
+ // Then handle specific error types
switch (type) {
case "validation_error":
// Handle validation errors