LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
from __future__ import annotations
from dataclasses import asdict, dataclass, fields
from typing import Any
from celery.result import AsyncResult
from workflow_manager.endpoint_v2.dto import DestinationConfig, SourceConfig
from workflow_manager.file_execution.models import WorkflowFileExecution
from workflow_manager.utils.workflow_log import WorkflowLog
from workflow_manager.workflow_v2.constants import WorkflowKey
@dataclass
class ProvisionalWorkflow:
result: str
output: dict[str, str]
cost_type: str
cost: str
time_taken: float
def __init__(self, input_dict: dict[str, Any]) -> None:
self.result = input_dict.get(WorkflowKey.PWF_RESULT, "")
self.output = input_dict.get(WorkflowKey.PWF_OUTPUT, {})
self.cost_type = input_dict.get(WorkflowKey.PWF_COST_TYPE, "")
self.cost = input_dict.get(WorkflowKey.PWF_COST, "")
self.time_taken = input_dict.get(WorkflowKey.PWF_TIME_TAKEN, 0.0)
@dataclass
class ExecutionResponse:
"""DTO representing the response from a workflow/tool execution.
Attributes:
workflow_id: UUID of the workflow that was executed.
execution_id: UUID of the specific execution run.
execution_status: Current status (e.g. "PENDING", "EXECUTING",
"COMPLETED", "ERROR").
log_id: Optional ID for the associated execution log stream.
status_api: Optional URL/path for polling execution status.
error: Human-readable error message if execution failed.
mode: Execution mode (e.g. "DEFAULT", "QUEUE").
result: Execution output — a list of per-file result dicts, each
containing:
- ``"file"`` (str): Source file identifier.
- ``"result"`` (dict): Extracted output with keys ``"output"``
(the structured extraction), ``"metadata"`` (highlight data,
per-model usage, confidence scores), and ``"metrics"``
(timing, token counts).
- ``"metadata"`` (dict): Workflow-level identifiers
(``source_name``, ``source_hash``, ``workflow_id``, etc.).
- ``"error"`` (str | None): Per-file error if extraction failed.
message: Optional human-readable status message.
result_acknowledged: Whether the caller has acknowledged/consumed
the result (used by async polling flows).
"""
workflow_id: str
execution_id: str
execution_status: str
log_id: str | None = None
status_api: str | None = None
error: str | None = None
mode: str | None = None
result: Any | None = None
message: str | None = None
result_acknowledged: bool = False
def __post_init__(self) -> None:
self.log_id = self.log_id or None
self.mode = self.mode or None
self.error = self.error or None
self.result = self.result or None
self.message = self.message or None
self.status_api = self.status_api or None
@staticmethod
def _remove_item_top_metadata(item: dict, keys_to_remove: list[str]) -> None:
"""Remove metadata keys from top-level item['metadata']."""
if "metadata" not in item:
return
if keys_to_remove:
item_metadata = item["metadata"]
if isinstance(item_metadata, dict):
for key in keys_to_remove:
item_metadata.pop(key, None)
else:
item.pop("metadata", None)
def remove_result_metadata_keys(self, keys_to_remove: list[str] = []) -> None:
"""Removes specified keys from the 'metadata' dictionary within each
'result' dictionary in the 'result' list attribute of the instance. If
'keys_to_remove' is empty, the 'metadata' key itself is removed.
Args:
keys_to_remove (List[str]): List of keys to be removed from 'metadata'.
"""
if not isinstance(self.result, list):
return
for item in self.result:
if not isinstance(item, dict):
continue
# Handle metadata nested inside item["result"]["metadata"]
result = item.get("result")
if isinstance(result, dict):
self._remove_specific_keys(result=result, keys_to_remove=keys_to_remove)
# Handle top-level item["metadata"] (workers cache path)
self._remove_item_top_metadata(item, keys_to_remove)
def remove_inner_result_metadata(self) -> None:
"""Removes only the inner item["result"]["metadata"] dict (extraction
metadata like highlight_data, per-model costs, etc.) while preserving
the outer item["metadata"] dict which contains workflow identification
keys (source_name, source_hash, workflow_id, etc.).
Use this instead of remove_result_metadata_keys() when you want to
strip extraction metadata but keep workflow identification metadata.
"""
if not isinstance(self.result, list):
return
for item in self.result:
if not isinstance(item, dict):
continue
result = item.get("result")
if isinstance(result, dict):
result.pop("metadata", None)
def promote_extracted_text(self) -> None:
"""Copies extracted_text from metadata to the top level of each file
result item. This allows extracted_text to be returned independently
of include_metadata.
After promotion, the extracted_text appears as:
result[i]["extracted_text"] = "..."
"""
if not isinstance(self.result, list):
return
for item in self.result:
if not isinstance(item, dict):
continue
result = item.get("result")
if not isinstance(result, dict):
continue
metadata = result.get("metadata", {})
extracted_text = metadata.get("extracted_text")
if extracted_text is not None:
item["extracted_text"] = extracted_text
def remove_result_metrics(self) -> None:
"""Removes the 'metrics' key from the 'result' dictionary within each
'result' dictionary in the 'result' list attribute of the instance.
"""
if not isinstance(self.result, list):
return
for item in self.result:
if not isinstance(item, dict):
continue
result = item.get("result")
if isinstance(result, dict):
result.pop("metrics", None)
def _remove_specific_keys(self, result: dict, keys_to_remove: list[str]) -> None:
"""Removes specified keys from the 'metadata' dictionary within the
provided 'result' dictionary. If 'keys_to_remove' is empty, the
'metadata' dictionary is cleared.
Args:
result (dict): The dictionary containing the 'metadata' key.
keys_to_remove (List[str]): List of keys to be removed from 'metadata'.
"""
metadata = result.get("metadata", {})
if keys_to_remove:
for key in keys_to_remove:
metadata.pop(key, None)
else:
metadata = {}
self._update_metadata(result=result, metadata=metadata)
def _update_metadata(self, result: dict, metadata: dict) -> None:
"""Updates the 'metadata' key in the provided 'result' dictionary. If
'metadata' is empty, removes the 'metadata' key from 'result'.
Args:
result (dict): The dictionary to be updated.
metadata (dict): The new metadata dictionary to be set. If empty, 'metadata'
is removed.
"""
if metadata:
result["metadata"] = metadata
else:
result.pop("metadata", None)
@dataclass
class AsyncResultData:
id: str
status: str
result: Any
is_ready: bool
is_failed: bool
info: Any
def __init__(self, async_result: AsyncResult):
self.id = async_result.id
self.status = async_result.status
self.result = async_result.result
self.is_ready = async_result.ready()
self.is_failed = async_result.failed()
self.info = async_result.info
if isinstance(self.result, Exception):
self.result = str(self.result)
def to_dict(self) -> dict[str, Any]:
return {
"status": self.status,
"result": self.result,
}
@dataclass
class FileData:
workflow_id: str
source_config: dict[str, Any]
destination_config: dict[str, Any]
execution_id: str
single_step: bool
organization_id: str
pipeline_id: str
scheduled: bool
execution_mode: str
use_file_history: bool
q_file_no_list: list[int]
llm_profile_id: str | None = None
custom_data: dict[str, Any] | None = None
@classmethod
def from_dict(cls, data: dict[str, Any]) -> FileData:
field_names = {f.name for f in fields(cls)}
filtered_data = {k: v for k, v in data.items() if k in field_names}
return cls(**filtered_data)
def __str__(self) -> str:
return f"FileData({self.__dict__})"
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@dataclass
class FileBatchData:
files: list
file_data: FileData
@classmethod
def from_dict(cls, data: dict[str, Any]) -> FileBatchData:
file_data = FileData.from_dict(data["file_data"])
return cls(files=data["files"], file_data=file_data)
def __str__(self) -> str:
return f"FileBatchData(files={self.files}, file_data={self.file_data})"
def to_dict(self) -> dict[str, Any]:
return asdict(self)
@dataclass
class FileBatchResult:
successful_files: int = 0
failed_files: int = 0
@property
def total_files(self) -> int:
return self.successful_files + self.failed_files
def to_dict(self) -> dict[str, int]:
return {
"successful_files": self.successful_files,
"failed_files": self.failed_files,
}
@dataclass
class ToolExecutionResult:
error: str | None
result: Any | None
def to_dict(self) -> dict[str, Any]:
return {
"error": self.error,
"result": self.result,
}
@dataclass
class FinalOutputResult:
output: Any | None
metadata: dict[str, Any] | None
error: str | None
def to_dict(self) -> dict[str, Any]:
return {
"output": self.output,
"metadata": self.metadata,
"error": self.error,
}
@dataclass
class ExecutionContext:
workflow_log: WorkflowLog
workflow_file_execution: WorkflowFileExecution
source_config: SourceConfig
destination_config: DestinationConfig