LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Logging and Monitoring Utilities for Workers
Provides structured logging, performance monitoring, and metrics collection for workers.
"""
import dataclasses
import functools
import inspect
import logging
import os
import sys
import time
import warnings
from collections.abc import Callable, Mapping, Sequence
from contextlib import contextmanager
from dataclasses import asdict, dataclass
from datetime import UTC, datetime
from threading import local
from typing import Any
from uuid import UUID
# Thread-local storage for context
_context = local()
@dataclass
class LogContext:
"""Context information for structured logging."""
worker_name: str | None = None
task_id: str | None = None
execution_id: str | None = None
organization_id: str | None = None
correlation_id: str | None = None
request_id: str | None = None
class RequestIDFilter(logging.Filter):
"""Filter to inject request_id into log records.
Resolution order:
1. ``record.request_id`` if set explicitly via ``extra={...}``.
2. ``LogContext.request_id`` from the thread-local context.
3. ``"-"`` placeholder.
"""
def filter(self, record):
value = getattr(record, "request_id", None)
if not value:
ctx = getattr(_context, "log_context", None)
value = getattr(ctx, "request_id", None) if ctx else None
record.request_id = value or "-"
return True
class OTelFieldFilter(logging.Filter):
"""Filter to inject OpenTelemetry fields into log records.
Adopts the proven pattern from unstract/core/flask/logging.py for consistency.
Normalizes missing or falsy values (None, empty string, etc.) to "-" for
consistent log formatting.
"""
def filter(self, record):
for attr in ["otelTraceID", "otelSpanID"]:
if not getattr(record, attr, None):
setattr(record, attr, "-")
return True
class WorkerLogger:
"""Enhanced logger for worker processes with structured logging and context management."""
_loggers: dict[str, logging.Logger] = {}
_configured = False
@classmethod
def configure(
cls,
log_level: str = "INFO",
log_file: str | None = None,
worker_name: str | None = None,
):
"""Configure global logging settings.
Uses a single standardized format matching the Django backend.
Filters ensure required fields are always present to prevent KeyError/SIGSEGV.
Args:
log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
log_file: Optional log file path
worker_name: Worker name for context
"""
if cls._configured:
return
# Set root logger level
root_logger = logging.getLogger()
root_logger.setLevel(getattr(logging, log_level.upper()))
# Clear existing handlers
root_logger.handlers.clear()
# Use single standardized format (same as Django backend)
formatter = logging.Formatter(
"%(levelname)s : [%(asctime)s]"
"{module:%(module)s process:%(process)d "
"thread:%(thread)d request_id:%(request_id)s "
"trace_id:%(otelTraceID)s span_id:%(otelSpanID)s} :- %(message)s"
)
# Console handler
console_handler = logging.StreamHandler(sys.stdout)
console_handler.setFormatter(formatter)
# ALWAYS add filters to ensure required fields are present
# This prevents KeyError/ValueError that cause SIGSEGV
console_handler.addFilter(RequestIDFilter())
console_handler.addFilter(OTelFieldFilter())
root_logger.addHandler(console_handler)
# File handler if specified
if log_file:
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(formatter)
# Add filters to file handler as well
file_handler.addFilter(RequestIDFilter())
file_handler.addFilter(OTelFieldFilter())
root_logger.addHandler(file_handler)
# Set worker context if provided
if worker_name:
cls.set_context(LogContext(worker_name=worker_name))
# Suppress only generic HTTP/network libraries that are universally noisy
# Cloud-specific SDK logging is handled by each connector
logging.getLogger("urllib3").setLevel(logging.WARNING)
logging.getLogger("requests").setLevel(logging.WARNING)
logging.getLogger("httpx").setLevel(logging.WARNING)
# Suppress OpenTelemetry EventLogger LogRecord deprecation warning
# This is a bug in OpenTelemetry SDK 1.37.0 where EventLogger.emit() still uses
# deprecated trace_id/span_id/trace_flags parameters instead of context parameter.
# See: https://github.com/open-telemetry/opentelemetry-python/issues/4328
# TODO: Remove this suppression once OpenTelemetry fixes EventLogger.emit()
warnings.filterwarnings(
"ignore",
message="LogRecord init with.*trace_id.*span_id.*trace_flags.*deprecated",
category=UserWarning,
)
_install_celery_request_id_signals()
cls._configured = True
@classmethod
def get_logger(cls, name: str) -> logging.Logger:
"""Get or create a logger instance.
Args:
name: Logger name (typically __name__)
Returns:
Logger instance
"""
if name not in cls._loggers:
logger = logging.getLogger(name)
cls._loggers[name] = logger
return cls._loggers[name]
@classmethod
def setup(cls, worker_type) -> logging.Logger:
"""Centralized setup for worker logging using WorkerType enum.
Args:
worker_type: WorkerType enum instance
Returns:
Configured logger for the worker
Note:
This method uses the WorkerRegistry to get logging configuration,
ensuring consistency across all workers.
"""
from shared.enums.worker_enums import WorkerType
from shared.worker_registry import WorkerRegistry
# Validate worker_type
if not isinstance(worker_type, WorkerType):
raise TypeError(f"Expected WorkerType enum, got {type(worker_type)}")
# Get logging config from registry
logging_config = WorkerRegistry.get_logging_config(worker_type)
# Configure logging with registry settings
cls.configure(
log_level=os.getenv("LOG_LEVEL", logging_config.get("log_level", "INFO")),
worker_name=worker_type.to_worker_name(),
)
# Return logger for the worker
return cls.get_logger(worker_type.to_worker_name())
@classmethod
def set_context(cls, context: LogContext):
"""Set thread-local logging context."""
_context.log_context = context
@classmethod
def update_context(cls, **kwargs):
"""Update thread-local logging context."""
if not hasattr(_context, "log_context"):
_context.log_context = LogContext()
for key, value in kwargs.items():
if hasattr(_context.log_context, key):
setattr(_context.log_context, key, value)
@classmethod
def clear_context(cls):
"""Clear thread-local logging context."""
if hasattr(_context, "log_context"):
delattr(_context, "log_context")
@classmethod
def get_context(cls) -> LogContext | None:
"""Get current thread-local logging context."""
return getattr(_context, "log_context", None)
@dataclass
class PerformanceMetrics:
"""Performance metrics for function execution."""
function_name: str
execution_time: float
success: bool
error_type: str | None = None
error_message: str | None = None
memory_usage: float | None = None
start_time: datetime | None = None
end_time: datetime | None = None
class PerformanceMonitor:
"""Performance monitoring utilities for tracking function execution metrics."""
def __init__(self, logger: logging.Logger | None = None):
"""Initialize performance monitor.
Args:
logger: Logger instance. Uses default if None.
"""
self.logger = logger or WorkerLogger.get_logger(__name__)
self.metrics: dict[str, list] = {}
def __call__(self, func: Callable) -> Callable:
"""Decorator to monitor function performance."""
@functools.wraps(func)
def wrapper(*args, **kwargs):
return self.monitor_execution(func, *args, **kwargs)
return wrapper
def monitor_execution(self, func: Callable, *args, **kwargs) -> Any:
"""Monitor function execution and collect metrics.
Args:
func: Function to monitor
*args: Function arguments
**kwargs: Function keyword arguments
Returns:
Function result
"""
start_time = time.time()
start_datetime = datetime.now(UTC)
function_name = f"{func.__module__}.{func.__name__}"
try:
# Get memory usage before execution (if psutil available)
memory_before = self._get_memory_usage()
self.logger.debug(f"Starting execution of {function_name}")
# Execute function
result = func(*args, **kwargs)
# Calculate metrics
end_time = time.time()
end_datetime = datetime.now(UTC)
execution_time = end_time - start_time
memory_after = self._get_memory_usage()
memory_usage = (
memory_after - memory_before if memory_before and memory_after else None
)
# Create metrics record
metrics = PerformanceMetrics(
function_name=function_name,
execution_time=execution_time,
success=True,
memory_usage=memory_usage,
start_time=start_datetime,
end_time=end_datetime,
)
# Log successful execution
self.logger.info(
f"Function {function_name} completed successfully",
extra={
"execution_time": execution_time,
"memory_usage": memory_usage,
"performance_metrics": asdict(metrics),
},
)
# Store metrics
self._store_metrics(metrics)
return result
except Exception as e:
# Calculate metrics for failed execution
end_time = time.time()
end_datetime = datetime.now(UTC)
execution_time = end_time - start_time
metrics = PerformanceMetrics(
function_name=function_name,
execution_time=execution_time,
success=False,
error_type=type(e).__name__,
error_message=str(e),
start_time=start_datetime,
end_time=end_datetime,
)
# Log failed execution
self.logger.error(
f"Function {function_name} failed after {execution_time:.3f}s",
extra={
"execution_time": execution_time,
"error_type": type(e).__name__,
"error_message": str(e),
"performance_metrics": asdict(metrics),
},
exc_info=True,
)
# Store metrics
self._store_metrics(metrics)
raise
def _get_memory_usage(self) -> float | None:
"""Get current memory usage in MB."""
try:
import psutil
process = psutil.Process(os.getpid())
return process.memory_info().rss / 1024 / 1024 # Convert to MB
except ImportError:
return None
except Exception:
return None
def _store_metrics(self, metrics: PerformanceMetrics):
"""Store metrics for later analysis."""
function_name = metrics.function_name
if function_name not in self.metrics:
self.metrics[function_name] = []
self.metrics[function_name].append(metrics)
# Keep only last 100 metrics per function to prevent memory bloat
if len(self.metrics[function_name]) > 100:
self.metrics[function_name] = self.metrics[function_name][-100:]
def get_metrics(self, function_name: str | None = None) -> dict[str, Any]:
"""Get performance metrics.
Args:
function_name: Specific function name. Returns all if None.
Returns:
Metrics dictionary
"""
if function_name:
function_metrics = self.metrics.get(function_name, [])
else:
function_metrics = []
for metrics_list in self.metrics.values():
function_metrics.extend(metrics_list)
if not function_metrics:
return {}
# Calculate aggregated metrics
total_executions = len(function_metrics)
successful_executions = sum(1 for m in function_metrics if m.success)
failed_executions = total_executions - successful_executions
execution_times = [m.execution_time for m in function_metrics]
avg_execution_time = sum(execution_times) / len(execution_times)
min_execution_time = min(execution_times)
max_execution_time = max(execution_times)
return {
"function_name": function_name,
"total_executions": total_executions,
"successful_executions": successful_executions,
"failed_executions": failed_executions,
"success_rate": successful_executions / total_executions
if total_executions > 0
else 0,
"avg_execution_time": avg_execution_time,
"min_execution_time": min_execution_time,
"max_execution_time": max_execution_time,
"recent_metrics": [
asdict(m) for m in function_metrics[-10:]
], # Last 10 executions
}
def get_summary(self) -> dict[str, Any]:
"""Get summary of all monitored functions."""
summary = {}
for function_name in self.metrics:
summary[function_name] = self.get_metrics(function_name)
return summary
@contextmanager
def log_context(**kwargs):
"""Context manager for temporary logging context.
Usage:
with log_context(task_id='123', execution_id='456'):
logger.info("This will include context")
"""
# Save current context
original_context = WorkerLogger.get_context()
try:
# Update context
WorkerLogger.update_context(**kwargs)
yield
finally:
# Restore original context
if original_context:
WorkerLogger.set_context(original_context)
else:
WorkerLogger.clear_context()
def with_execution_context(func: Callable) -> Callable:
"""Decorator to automatically set up logging context for execution functions.
This decorator extracts common execution context parameters from function arguments
and sets up logging context automatically, eliminating the need for repetitive
log_context() usage in task functions.
Expected function signature patterns:
- func(self, schema_name, workflow_id, execution_id, pipeline_id=None, ...)
- func(self, schema_name, execution_id, workflow_id, organization_id=None, ...)
Args:
func: The function to wrap with execution context
Returns:
Wrapped function with automatic logging context setup
Usage:
@with_execution_context
def my_task(self, schema_name: str, workflow_id: str, execution_id: str, ...):
# Function body - logging context is automatically available
logger.info("This will include task_id, execution_id, etc.")
"""
from functools import wraps
@wraps(func)
def wrapper(task_instance, *args, **kwargs) -> Any:
# Extract task_id from Celery task instance
task_id = (
getattr(task_instance.request, "id", None)
if hasattr(task_instance, "request")
else None
)
# Extract common context parameters from arguments
# Handle different argument patterns used across tasks
context = {"task_id": task_id}
if args:
# Most common pattern: (schema_name, workflow_id, execution_id, ...)
if len(args) >= 1:
schema_name = args[0]
context["organization_id"] = schema_name
if len(args) >= 2:
# Could be workflow_id or execution_id depending on task
if len(args) >= 3:
# Pattern: (schema_name, workflow_id, execution_id, ...)
context["workflow_id"] = args[1]
context["execution_id"] = args[2]
else:
# Pattern: (schema_name, execution_id, ...)
context["execution_id"] = args[1]
if len(args) >= 4:
# Check if 4th argument is pipeline_id or another param
pipeline_id = (
args[3] if args[3] is not None else kwargs.get("pipeline_id")
)
if pipeline_id:
context["pipeline_id"] = pipeline_id
# Extract additional context from kwargs
for key in ["workflow_id", "execution_id", "pipeline_id", "organization_id"]:
if key in kwargs and kwargs[key] is not None:
context[key] = kwargs[key]
# Remove None values from context
context = {k: v for k, v in context.items() if v is not None}
# Execute function with logging context
with log_context(**context):
return func(task_instance, *args, **kwargs)
return wrapper
def logged_execution(logger: logging.Logger | None = None):
"""Decorator for automatic logging of function execution.
Args:
logger: Logger instance. Uses default if None.
Returns:
Decorated function
"""
def decorator(func: Callable) -> Callable:
actual_logger = logger or WorkerLogger.get_logger(func.__module__)
@functools.wraps(func)
def wrapper(*args, **kwargs):
function_name = f"{func.__module__}.{func.__name__}"
actual_logger.debug(f"Starting execution of {function_name}")
try:
result = func(*args, **kwargs)
actual_logger.debug(f"Successfully completed {function_name}")
return result
except Exception as e:
actual_logger.error(
f"Function {function_name} failed: {e}", exc_info=True
)
raise
return wrapper
return decorator
# Create global performance monitor instance
performance_monitor = PerformanceMonitor()
# Convenience decorators
def monitor_performance(func: Callable) -> Callable:
"""Decorator for performance monitoring."""
return performance_monitor(func)
def log_execution(func: Callable) -> Callable:
"""Decorator for execution logging."""
return logged_execution()(func)
_REQUEST_ID_KEYS: tuple[str, ...] = (
"request_id",
"file_execution_id",
"execution_id",
"run_id",
)
def _coerce_id(value: Any) -> str | None:
"""Return a safe string id, or None if value is not a usable id type.
Rejects arbitrary objects so a misnamed payload field (e.g. a dataclass
or a list) cannot leak its ``__str__`` into log lines or OTel attributes.
Booleans are excluded explicitly because ``bool`` is a subclass of
``int`` in Python; without the guard, ``True``/``False`` would
serialize to the literal strings ``"True"``/``"False"``.
"""
if isinstance(value, bool):
return None
if isinstance(value, str):
return value or None
if isinstance(value, (int, UUID)):
return str(value)
return None
def _bind_positional_args_to_names(
args: Sequence[Any], kwargs: Mapping[str, Any], task: Any
) -> Mapping[str, Any] | None:
"""Map positional args to parameter names via the task's signature.
Catches the case where ids are passed as positional strings (e.g.
``send_task("async_execute_bin", args=[schema, wf_id, exec_id, ...])``).
Returns ``None`` if the task is not introspectable or has no resolved
arguments.
"""
if task is None:
return None
runner = getattr(task, "run", task)
try:
bound = inspect.signature(runner).bind_partial(*(args or ()), **(kwargs or {}))
except (TypeError, ValueError):
return None
return dict(bound.arguments) if bound.arguments else None
def _arg_as_mapping(arg: Any) -> Mapping[str, Any] | None:
"""Return a mapping view of a positional arg, or None if not coercible.
Accepts ``Mapping`` instances directly and dataclass instances via
``dataclasses.asdict``.
"""
if isinstance(arg, Mapping):
return arg
if dataclasses.is_dataclass(arg) and not isinstance(arg, type):
return dataclasses.asdict(arg)
return None
def _gather_containers(
args: Sequence[Any], kwargs: Mapping[str, Any], task: Any
) -> list[Mapping[str, Any]]:
"""Build the ordered list of mappings to scan for an id.
Order (used only for stable iteration; final priority is by KEY in
``_extract_request_id``):
1. Positional args bound to parameter names via the task signature.
2. Top-level kwargs.
3. ``Mapping`` values nested inside kwargs.
4. ``Mapping`` / dataclass positional args.
"""
containers: list[Mapping[str, Any]] = []
bound = _bind_positional_args_to_names(args, kwargs, task)
if bound:
containers.append(bound)
if kwargs:
containers.append(kwargs)
containers.extend(v for v in kwargs.values() if isinstance(v, Mapping))
for arg in args or ():
mapping = _arg_as_mapping(arg)
if mapping is not None:
containers.append(mapping)
return containers
def _extract_request_id(
args: Sequence[Any], kwargs: Mapping[str, Any], task: Any = None
) -> str | None:
"""Pull a usable request_id from a Celery task payload.
Workers bind ``file_execution_id`` to the ``request_id`` log field
(legacy structure-tool convention), giving per-file granularity in
logs across the multi-tool execution chain. Cross-service correlation
across the API/HTTP boundary is handled by OpenTelemetry ``trace_id``,
not by ``request_id``.
The key priority below is also the migration path: callers may start
passing a real HTTP ``request_id`` in the payload at any time and it
will take precedence over ``file_execution_id`` automatically -- no
worker change required.
Priority is by KEY (not by container), so a payload with both
``request_id`` in one container and ``file_execution_id`` in another
deterministically picks ``request_id`` regardless of insertion order.
"""
containers = _gather_containers(args, kwargs, task)
for key in _REQUEST_ID_KEYS:
for container in containers:
value = _coerce_id(container.get(key))
if value:
return value
return None
def _bind_task_context(task_id, task, args, kwargs, **_):
"""Celery ``task_prerun`` handler: bind request_id onto the log context.
Catches any extraction failure so a malformed payload can never leave
the previous task's id bound on the thread.
"""
try:
request_id = _extract_request_id(args or (), kwargs or {}, task) or task_id
except Exception:
logging.getLogger(__name__).debug(
"request_id extraction failed for task %s; falling back to task_id",
task_id,
exc_info=True,
)
request_id = task_id
WorkerLogger.update_context(request_id=request_id, task_id=task_id)
def _clear_task_context(**_):
"""Celery ``task_postrun`` handler: reset task-scoped fields only.
Preserves baseline context (``worker_name``, etc.) set at
``WorkerLogger.configure()``; only nulls out the per-task fields bound
in ``_bind_task_context``.
"""
WorkerLogger.update_context(request_id=None, task_id=None)
@functools.lru_cache(maxsize=1)
def _install_celery_request_id_signals() -> None:
"""Wire Celery signals once per process.
Idempotent and thread-safe via ``functools.lru_cache``. No-ops with a
debug log if Celery is not importable (e.g. unit tests).
"""
try:
from celery.signals import task_postrun, task_prerun
except ImportError as exc:
logging.getLogger(__name__).debug(
"celery.signals not importable; request_id signal install skipped: %s",
exc,
)
return
task_prerun.connect(_bind_task_context, weak=False)
task_postrun.connect(_clear_task_context, weak=False)
# Dataclass/Dictionary Access Utilities
def safe_get_attr(obj: Any, key: str, default: Any = None) -> Any:
"""Safely get value from object using both attribute and dictionary access patterns.
This utility handles the common pattern where objects may be either dataclass instances
or dictionaries, providing consistent access regardless of the actual type.
Args:
obj: Object to get value from (dataclass instance or dictionary)
key: Key/attribute name to access
default: Default value if key/attribute not found
Returns:
Value from object or default if not found
Examples:
>>> safe_get_attr(WorkerFileData(execution_id="123"), "execution_id")
"123"
>>> safe_get_attr({"execution_id": "456"}, "execution_id")
"456"
>>> safe_get_attr({}, "missing_key", "default")
"default"
"""
# First try attribute access (for dataclass objects)
if hasattr(obj, key):
return getattr(obj, key, default)
# Then try dictionary access
elif isinstance(obj, dict):
return obj.get(key, default)
# Fallback to default
else:
return default
def ensure_dict_access(obj: Any, keys: list, default: Any = None) -> dict[str, Any]:
"""Extract multiple values from object using safe access patterns.
Args:
obj: Object to extract values from
keys: List of keys/attributes to extract
default: Default value for missing keys
Returns:
Dictionary with extracted values
Example:
>>> ensure_dict_access(file_data, ["execution_id", "workflow_id", "organization_id"])
{'execution_id': '123', 'workflow_id': '456', 'organization_id': 'org_789'}
"""
return {key: safe_get_attr(obj, key, default) for key in keys}