Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

UN-2693 [FEAT] added raw view support backend oss support functionality (#1483)

vishnuszipstackff82d1678ae9
diff --git a/backend/workflow_manager/endpoint_v2/destination.py b/backend/workflow_manager/endpoint_v2/destination.py
index c94aedb13..8bcd7bdfc 100644
--- a/backend/workflow_manager/endpoint_v2/destination.py
+++ b/backend/workflow_manager/endpoint_v2/destination.py
@@ -532,14 +532,22 @@ class DestinationConnector(BaseConnector):
return result
def has_valid_metadata(self, metadata: Any) -> bool:
- # Check if metadata is not None and metadata is a non-empty string
- if not metadata:
+ # Check if metadata is not None and is either a non-empty dict or valid string
+ if metadata is None:
return False
- if not isinstance(metadata, str):
- return False
- if metadata.strip().lower() == "none":
- return False
- return True
+
+ # Handle dict metadata (which is valid and contains extracted_text)
+ if isinstance(metadata, dict):
+ return bool(metadata) # Return True if dict is not empty
+
+ # Handle string metadata
+ if isinstance(metadata, str):
+ if metadata.strip().lower() == "none" or not metadata.strip():
+ return False
+ return True
+
+ # For other types, consider them valid if they're truthy
+ return bool(metadata)
def get_metadata(
self, file_history: FileHistory | None = None
@@ -554,8 +562,18 @@ class DestinationConnector(BaseConnector):
return self.parse_string(file_history.metadata)
else:
return None
- metadata: dict[str, Any] = self.get_workflow_metadata()
+ # Try to get metadata from tool execution result first
+ try:
+ tool_result = self.get_tool_execution_result(file_history)
+ if isinstance(tool_result, dict) and "metadata" in tool_result:
+ # Return the metadata from tool execution result (includes extracted_text)
+ return tool_result.get("metadata", {})
+ except Exception as e:
+ logger.debug(f"Could not get metadata from tool result: {e}")
+
+ # Fallback to workflow metadata if tool result metadata not available
+ metadata: dict[str, Any] = self.get_workflow_metadata()
return metadata
def delete_file_execution_directory(self) -> None:
@@ -777,6 +795,9 @@ class DestinationConnector(BaseConnector):
q_name = self._get_review_queue_name()
whisper_hash = meta_data.get("whisper-hash") if meta_data else None
+ # Get extracted text from metadata (added by structure tool)
+ extracted_text = meta_data.get("extracted_text") if meta_data else None
+
queue_result = QueueResult(
file=file_name,
status=QueueResultStatus.SUCCESS,
@@ -785,6 +806,7 @@ class DestinationConnector(BaseConnector):
file_content=file_content_base64,
whisper_hash=whisper_hash,
file_execution_id=file_execution_id,
+ extracted_text=extracted_text,
).to_dict()
queue_result_json = json.dumps(queue_result)
@@ -811,6 +833,9 @@ class DestinationConnector(BaseConnector):
else:
whisper_hash = None
+ # Get extracted text from metadata (added by structure tool)
+ extracted_text = meta_data.get("extracted_text") if meta_data else None
+
# Create QueueResult with TTL metadata
queue_result_obj = QueueResult(
file=file_name,
@@ -820,6 +845,7 @@ class DestinationConnector(BaseConnector):
file_content=file_content_base64,
whisper_hash=whisper_hash,
file_execution_id=file_execution_id,
+ extracted_text=extracted_text,
)
# Add TTL metadata based on HITLSettings
diff --git a/backend/workflow_manager/endpoint_v2/queue_utils.py b/backend/workflow_manager/endpoint_v2/queue_utils.py
index d29660762..cda73fbe1 100644
--- a/backend/workflow_manager/endpoint_v2/queue_utils.py
+++ b/backend/workflow_manager/endpoint_v2/queue_utils.py
@@ -97,6 +97,7 @@ class QueueResult:
file_execution_id: str | None = None
enqueued_at: float | None = None
ttl_seconds: int | None = None
+ extracted_text: str | None = None
def __post_init__(self):
"""Initialize enqueued_at timestamp if not provided and validate required fields"""
@@ -122,5 +123,6 @@ class QueueResult:
"file_execution_id": self.file_execution_id,
"enqueued_at": self.enqueued_at,
"ttl_seconds": self.ttl_seconds,
+ "extracted_text": self.extracted_text,
}
return result_dict
diff --git a/tools/structure/src/main.py b/tools/structure/src/main.py
index b99f6a87a..ca7a7cdcd 100644
--- a/tools/structure/src/main.py
+++ b/tools/structure/src/main.py
@@ -325,11 +325,25 @@ class StructureTool(BaseTool):
)
# HACK: Replacing actual file's name instead of INFILE
- if SettingsKeys.METADATA in structured_output:
- structured_output[SettingsKeys.METADATA][SettingsKeys.FILE_NAME] = (
- self.source_file_name
- )
+ # Ensure metadata section exists
+ if SettingsKeys.METADATA not in structured_output:
+ structured_output[SettingsKeys.METADATA] = {}
+ self.stream_log("Created metadata section in structured_output")
+
+ structured_output[SettingsKeys.METADATA][SettingsKeys.FILE_NAME] = (
+ self.source_file_name
+ )
+ # Add extracted text for HITL raw view
+ if extracted_text:
+ structured_output[SettingsKeys.METADATA]["extracted_text"] = extracted_text
+ self.stream_log(
+ f"Added extracted_text to metadata (length: {len(extracted_text)} characters)"
+ )
+ else:
+ self.stream_log(
+ "No extracted_text available - STHelper.dynamic_extraction returned empty"
+ )
if merged_metrics := self._merge_metrics(
structured_output.get(SettingsKeys.METRICS, {}), index_metrics
):