LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Cache Utilities
Common utility functions for cache operations.
"""
import logging
from dataclasses import asdict, is_dataclass
from datetime import datetime
from enum import Enum
from typing import Any
logger = logging.getLogger(__name__)
def make_json_serializable(obj: Any) -> Any:
"""Convert an object to JSON-serializable format.
Handles common non-serializable types like dataclasses, enums, and datetime objects.
This is the single source of truth for JSON serialization in the cache system.
Args:
obj: Object to serialize
Returns:
JSON-serializable version of the object with type information for reconstruction
"""
def serialize_item(item):
if item is None:
return None
elif isinstance(item, (str, int, float, bool)):
return item
elif isinstance(item, (list, tuple)):
return [serialize_item(i) for i in item]
elif isinstance(item, dict):
return {k: serialize_item(v) for k, v in item.items()}
elif isinstance(item, Enum):
return item.value
elif isinstance(item, datetime):
return item.isoformat()
elif is_dataclass(item):
# Store type information for reconstruction
serialized = serialize_item(asdict(item))
return {
"__type__": f"{item.__class__.__module__}.{item.__class__.__name__}",
"__data__": serialized,
}
elif hasattr(item, "__dict__"):
# Handle objects with __dict__ (like API response objects)
serialized = serialize_item(item.__dict__)
return {
"__type__": f"{item.__class__.__module__}.{item.__class__.__name__}",
"__data__": serialized,
}
else:
# Fallback - convert to string
return str(item)
try:
return serialize_item(obj)
except Exception as e:
logger.warning(f"Failed to serialize object for caching: {e}")
# Return a simple representation that can be cached
return {"error": "serialization_failed", "type": str(type(obj))}
def reconstruct_from_cache(cached_data: Any) -> Any:
"""Reconstruct objects from cached data using type information.
Args:
cached_data: Data retrieved from cache
Returns:
Reconstructed object or original data if no type info
"""
def reconstruct_item(item):
if item is None:
return None
elif isinstance(item, (str, int, float, bool)):
return item
elif isinstance(item, list):
return [reconstruct_item(i) for i in item]
elif isinstance(item, dict):
# Check if this is a typed object
if "__type__" in item and "__data__" in item:
return reconstruct_typed_object(item["__type__"], item["__data__"])
else:
return {k: reconstruct_item(v) for k, v in item.items()}
else:
return item
try:
return reconstruct_item(cached_data)
except Exception as e:
logger.warning(f"Failed to reconstruct object from cache: {e}")
return cached_data
def reconstruct_typed_object(type_name: str, data: Any) -> Any:
"""Reconstruct a typed object from cached data.
Args:
type_name: Full type name (module.ClassName)
data: Serialized object data
Returns:
Reconstructed object or original data if reconstruction fails
"""
try:
# Registry of reconstructable types - cleaner than hardcoded strings
type_registry = _get_type_registry()
# Extract class name from full type path
class_name = type_name.split(".")[-1]
# Find matching class in registry
if class_name in type_registry:
cls = type_registry[class_name]
# Prefer from_dict method if available (handles nested objects properly)
if hasattr(cls, "from_dict") and callable(cls.from_dict):
logger.debug(f"Using from_dict method for {class_name}")
return cls.from_dict(data)
else:
# Fall back to direct constructor
logger.debug(f"Using direct constructor for {class_name}")
return cls(**data)
else:
logger.debug(f"Unknown type for reconstruction: {type_name}")
return data
except Exception as e:
logger.warning(f"Failed to reconstruct {type_name}: {e}")
return data
def _get_type_registry() -> dict[str, type]:
"""Get registry of reconstructable types.
Returns:
Dictionary mapping class names to classes
"""
try:
# Import core data models that are being cached
from unstract.core.data_models import (
ConnectorInstanceData,
TagData,
WorkflowDefinitionResponseData,
WorkflowEndpointConfigData,
WorkflowEndpointConfigResponseData,
WorkflowExecutionData,
)
from ..data.response_models import APIResponse
from ..models.api_responses import (
FileBatchResponse,
FileExecutionResponse,
FileHistoryResponse,
ManualReviewResponse,
ToolExecutionResponse,
ToolInstancesResponse,
WorkflowDefinitionResponse,
WorkflowEndpointsResponse,
WorkflowExecutionResponse,
)
return {
# Standard API response models
"APIResponse": APIResponse,
"WorkflowEndpointsResponse": WorkflowEndpointsResponse,
"ToolInstancesResponse": ToolInstancesResponse,
"WorkflowDefinitionResponse": WorkflowDefinitionResponse,
"WorkflowExecutionResponse": WorkflowExecutionResponse,
"FileExecutionResponse": FileExecutionResponse,
"ManualReviewResponse": ManualReviewResponse,
"FileBatchResponse": FileBatchResponse,
"ToolExecutionResponse": ToolExecutionResponse,
"FileHistoryResponse": FileHistoryResponse,
# Core data models that are cached
"WorkflowEndpointConfigResponseData": WorkflowEndpointConfigResponseData,
"WorkflowEndpointConfigData": WorkflowEndpointConfigData,
"ConnectorInstanceData": ConnectorInstanceData,
"WorkflowDefinitionResponseData": WorkflowDefinitionResponseData,
"TagData": TagData,
"WorkflowExecutionData": WorkflowExecutionData,
}
except ImportError as e:
logger.warning(f"Failed to import response types for cache reconstruction: {e}")
return {}