LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Worker Result Models
Dataclasses for task execution results.
"""
import logging
import os
import sys
import time
from dataclasses import dataclass, field
from typing import Any
# Import shared domain models from core
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "../../../unstract/core/src"))
from shared.enums import QueueResultStatus
from unstract.core import ExecutionStatus, serialize_dataclass_to_dict
from unstract.core.worker_models import FileExecutionResult
# Import worker enums
from ..enums import WebhookStatus
logger = logging.getLogger(__name__)
@dataclass
class WebhookResult:
"""Structured result for webhook delivery tasks."""
status: WebhookStatus
url: str
task_id: str
webhook_task_id: str
webhook_status: str
payload_size: int
timeout: int
attempts: int
delivery_time: float
error_message: str | None = None
response_code: int | None = None
response_body: str | None = None
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary for API response."""
return serialize_dataclass_to_dict(self)
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "WebhookResult":
"""Create from dictionary (e.g., task result)."""
return cls(
status=WebhookStatus(data.get("status", WebhookStatus.FAILED)),
url=data.get("url", ""),
task_id=data.get("task_id", ""),
webhook_task_id=data.get("webhook_task_id", ""),
webhook_status=data.get("webhook_status", ""),
payload_size=data.get("payload_size", 0),
timeout=data.get("timeout", 30),
attempts=data.get("attempts", 1),
delivery_time=data.get("delivery_time", 0.0),
error_message=data.get("error_message"),
response_code=data.get("response_code"),
response_body=data.get("response_body"),
)
@dataclass
class BatchExecutionResult:
"""Structured result for batch execution tasks."""
total_files: int
successful_files: int
failed_files: int
execution_time: float
file_results: list[FileExecutionResult] = field(default_factory=list)
batch_id: str | None = None
errors: list[str] = field(default_factory=list)
@property
def success_rate(self) -> float:
"""Calculate success rate as percentage."""
if self.total_files == 0:
return 0.0
return (self.successful_files / self.total_files) * 100
def to_dict(self) -> dict[str, Any]:
"""Convert to dictionary for API response."""
return serialize_dataclass_to_dict(self)
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "BatchExecutionResult":
"""Create from dictionary (e.g., task result)."""
file_results = [
FileExecutionResult.from_dict(result)
for result in data.get("file_results", [])
]
return cls(
total_files=data.get("total_files", 0),
successful_files=data.get("successful_files", 0),
failed_files=data.get("failed_files", 0),
execution_time=data.get("execution_time", 0.0),
file_results=file_results,
batch_id=data.get("batch_id"),
errors=data.get("errors", []),
)
def add_file_result(self, file_result: FileExecutionResult):
"""Add a file execution result to the batch."""
self.file_results.append(file_result)
self.total_files = len(self.file_results)
if file_result.is_successful():
self.successful_files += 1
else:
self.failed_files += 1
self.execution_time += file_result.processing_time
@dataclass
class CallbackProcessingResult:
"""Result for callback processing operations."""
callback_id: str
execution_id: str
organization_id: str
workflow_id: str
results: list[dict[str, Any]] = field(default_factory=list)
callback_status: ExecutionStatus = ExecutionStatus.COMPLETED
processing_time: float = 0.0
successful_callbacks: int = 0
failed_callbacks: int = 0
error_message: str | None = None
metadata: dict[str, Any] | None = None
@property
def result_count(self) -> int:
"""Get the number of results in this callback."""
return len(self.results) if self.results else 0
@property
def has_results(self) -> bool:
"""Check if this callback has any results."""
return self.result_count > 0
@property
def is_successful(self) -> bool:
"""Check if callback processing was successful."""
return (
self.callback_status == ExecutionStatus.COMPLETED and not self.error_message
)
def get_successful_results(self) -> list[dict[str, Any]]:
"""Filter and return only successful results."""
if not self.results:
return []
return [
result
for result in self.results
if isinstance(result, dict) and not result.get("error")
]
def get_failed_results(self) -> list[dict[str, Any]]:
"""Filter and return only failed results."""
if not self.results:
return []
return [
result
for result in self.results
if isinstance(result, dict) and result.get("error")
]
def to_dict(self) -> dict[str, Any]:
"""Convert callback result to dictionary."""
return serialize_dataclass_to_dict(self)
@classmethod
def from_dict(cls, data: dict[str, Any]) -> "CallbackProcessingResult":
"""Create CallbackProcessingResult from dictionary data."""
status_str = data.get("callback_status", ExecutionStatus.COMPLETED.value)
status = (
ExecutionStatus(status_str) if isinstance(status_str, str) else status_str
)
return cls(
callback_id=data["callback_id"],
execution_id=data["execution_id"],
organization_id=data["organization_id"],
workflow_id=data["workflow_id"],
results=data.get("results", []),
callback_status=status,
processing_time=data.get("processing_time", 0.0),
successful_callbacks=data.get("successful_callbacks", 0),
failed_callbacks=data.get("failed_callbacks", 0),
error_message=data.get("error_message"),
metadata=data.get("metadata"),
)
# Utility functions for result aggregation
def aggregate_file_results(
file_results: list[FileExecutionResult],
) -> BatchExecutionResult:
"""Aggregate multiple file results into a batch result."""
successful = len([r for r in file_results if r.is_successful()])
failed = len([r for r in file_results if r.has_error()])
total_time = sum(r.processing_time for r in file_results)
return BatchExecutionResult(
total_files=len(file_results),
successful_files=successful,
failed_files=failed,
execution_time=total_time,
file_results=file_results,
)
@dataclass
class QueueResult:
file: str
status: QueueResultStatus
result: Any
workflow_id: str
file_content: str | None = None
whisper_hash: str | None = None
file_execution_id: str | None = None
# Workflow execution id — carried into the HITL queue message so the
# manual-review consumer (pluggable_apps.manual_review_v2, a separate
# codebase) can populate hitl_queue.execution_id. The durable local fact is
# that to_dict() always emits this key (None when unset); the key name must
# stay in sync with that consumer. Optional with a None default because the
# connector's execution_id is itself nullable on some paths — a missing
# value is logged in __post_init__ rather than raised, so an unexpected NULL
# is visible without breaking the HITL enqueue.
execution_id: str | None = None
enqueued_at: float | None = None
ttl_seconds: int | None = None
extracted_text: str | None = None
hitl_reason: str | None = None
hitl_queue_name: str | None = None
def __post_init__(self):
"""Initialize enqueued_at timestamp if not provided and validate required fields"""
if self.enqueued_at is None:
self.enqueued_at = time.time()
# Validate required fields
if not self.file:
raise ValueError("QueueResult requires a valid file name")
if not self.workflow_id:
raise ValueError("QueueResult requires a valid workflow_id")
if self.status is None:
raise ValueError("QueueResult requires a valid status")
# execution_id should always be set on the HITL enqueue path; it's not a
# hard requirement (the connector's value is nullable on some paths and a
# raise would break manual-review enqueue), so surface a missing value
# rather than silently writing a NULL hitl_queue.execution_id.
if not self.execution_id:
logger.warning(
"QueueResult for file=%r has no execution_id; "
"hitl_queue.execution_id will be NULL",
self.file,
)
def to_dict(self) -> Any:
result_dict = {
"file": self.file,
"whisper_hash": self.whisper_hash,
"status": self.status.value,
"result": self.result,
"workflow_id": self.workflow_id,
"file_content": self.file_content,
"file_execution_id": self.file_execution_id,
"execution_id": self.execution_id,
"enqueued_at": self.enqueued_at,
"ttl_seconds": self.ttl_seconds,
"extracted_text": self.extracted_text,
"hitl_reason": self.hitl_reason,
"hitl_queue_name": self.hitl_queue_name,
}
return result_dict