Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

26.4 KB
Raw Blame
"""Logging and Monitoring Utilities for Workers

Provides structured logging, performance monitoring, and metrics collection for workers.
"""

import dataclasses
import functools
import inspect
import logging
import os
import sys
import time
import warnings
from collections.abc import Callable, Mapping, Sequence
from contextlib import contextmanager
from dataclasses import asdict, dataclass
from datetime import UTC, datetime
from threading import local
from typing import Any
from uuid import UUID

# Thread-local storage for context
_context = local()


@dataclass
class LogContext:
    """Context information for structured logging."""

    worker_name: str | None = None
    task_id: str | None = None
    execution_id: str | None = None
    organization_id: str | None = None
    correlation_id: str | None = None
    request_id: str | None = None


class RequestIDFilter(logging.Filter):
    """Filter to inject request_id into log records.

    Resolution order:
      1. ``record.request_id`` if set explicitly via ``extra={...}``.
      2. ``LogContext.request_id`` from the thread-local context.
      3. ``"-"`` placeholder.
    """

    def filter(self, record):
        value = getattr(record, "request_id", None)
        if not value:
            ctx = getattr(_context, "log_context", None)
            value = getattr(ctx, "request_id", None) if ctx else None
        record.request_id = value or "-"
        return True


class OTelFieldFilter(logging.Filter):
    """Filter to inject OpenTelemetry fields into log records.

    Adopts the proven pattern from unstract/core/flask/logging.py for consistency.
    Normalizes missing or falsy values (None, empty string, etc.) to "-" for
    consistent log formatting.
    """

    def filter(self, record):
        for attr in ["otelTraceID", "otelSpanID"]:
            if not getattr(record, attr, None):
                setattr(record, attr, "-")
        return True


class WorkerLogger:
    """Enhanced logger for worker processes with structured logging and context management."""

    _loggers: dict[str, logging.Logger] = {}
    _configured = False

    @classmethod
    def configure(
        cls,
        log_level: str = "INFO",
        log_file: str | None = None,
        worker_name: str | None = None,
    ):
        """Configure global logging settings.

        Uses a single standardized format matching the Django backend.
        Filters ensure required fields are always present to prevent KeyError/SIGSEGV.

        Args:
            log_level: Logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
            log_file: Optional log file path
            worker_name: Worker name for context
        """
        if cls._configured:
            return

        # Set root logger level
        root_logger = logging.getLogger()
        root_logger.setLevel(getattr(logging, log_level.upper()))

        # Clear existing handlers
        root_logger.handlers.clear()

        # Use single standardized format (same as Django backend)
        formatter = logging.Formatter(
            "%(levelname)s : [%(asctime)s]"
            "{module:%(module)s process:%(process)d "
            "thread:%(thread)d request_id:%(request_id)s "
            "trace_id:%(otelTraceID)s span_id:%(otelSpanID)s} :- %(message)s"
        )

        # Console handler
        console_handler = logging.StreamHandler(sys.stdout)
        console_handler.setFormatter(formatter)

        # ALWAYS add filters to ensure required fields are present
        # This prevents KeyError/ValueError that cause SIGSEGV
        console_handler.addFilter(RequestIDFilter())
        console_handler.addFilter(OTelFieldFilter())

        root_logger.addHandler(console_handler)

        # File handler if specified
        if log_file:
            file_handler = logging.FileHandler(log_file)
            file_handler.setFormatter(formatter)

            # Add filters to file handler as well
            file_handler.addFilter(RequestIDFilter())
            file_handler.addFilter(OTelFieldFilter())

            root_logger.addHandler(file_handler)

        # Set worker context if provided
        if worker_name:
            cls.set_context(LogContext(worker_name=worker_name))

        # Suppress only generic HTTP/network libraries that are universally noisy
        # Cloud-specific SDK logging is handled by each connector
        logging.getLogger("urllib3").setLevel(logging.WARNING)
        logging.getLogger("requests").setLevel(logging.WARNING)
        logging.getLogger("httpx").setLevel(logging.WARNING)

        # Suppress OpenTelemetry EventLogger LogRecord deprecation warning
        # This is a bug in OpenTelemetry SDK 1.37.0 where EventLogger.emit() still uses
        # deprecated trace_id/span_id/trace_flags parameters instead of context parameter.
        # See: https://github.com/open-telemetry/opentelemetry-python/issues/4328
        # TODO: Remove this suppression once OpenTelemetry fixes EventLogger.emit()
        warnings.filterwarnings(
            "ignore",
            message="LogRecord init with.*trace_id.*span_id.*trace_flags.*deprecated",
            category=UserWarning,
        )

        _install_celery_request_id_signals()

        cls._configured = True

    @classmethod
    def get_logger(cls, name: str) -> logging.Logger:
        """Get or create a logger instance.

        Args:
            name: Logger name (typically __name__)

        Returns:
            Logger instance
        """
        if name not in cls._loggers:
            logger = logging.getLogger(name)
            cls._loggers[name] = logger

        return cls._loggers[name]

    @classmethod
    def setup(cls, worker_type) -> logging.Logger:
        """Centralized setup for worker logging using WorkerType enum.

        Args:
            worker_type: WorkerType enum instance

        Returns:
            Configured logger for the worker

        Note:
            This method uses the WorkerRegistry to get logging configuration,
            ensuring consistency across all workers.
        """
        from shared.enums.worker_enums import WorkerType
        from shared.worker_registry import WorkerRegistry

        # Validate worker_type
        if not isinstance(worker_type, WorkerType):
            raise TypeError(f"Expected WorkerType enum, got {type(worker_type)}")

        # Get logging config from registry
        logging_config = WorkerRegistry.get_logging_config(worker_type)

        # Configure logging with registry settings
        cls.configure(
            log_level=os.getenv("LOG_LEVEL", logging_config.get("log_level", "INFO")),
            worker_name=worker_type.to_worker_name(),
        )

        # Return logger for the worker
        return cls.get_logger(worker_type.to_worker_name())

    @classmethod
    def set_context(cls, context: LogContext):
        """Set thread-local logging context."""
        _context.log_context = context

    @classmethod
    def update_context(cls, **kwargs):
        """Update thread-local logging context."""
        if not hasattr(_context, "log_context"):
            _context.log_context = LogContext()

        for key, value in kwargs.items():
            if hasattr(_context.log_context, key):
                setattr(_context.log_context, key, value)

    @classmethod
    def clear_context(cls):
        """Clear thread-local logging context."""
        if hasattr(_context, "log_context"):
            delattr(_context, "log_context")

    @classmethod
    def get_context(cls) -> LogContext | None:
        """Get current thread-local logging context."""
        return getattr(_context, "log_context", None)


@dataclass
class PerformanceMetrics:
    """Performance metrics for function execution."""

    function_name: str
    execution_time: float
    success: bool
    error_type: str | None = None
    error_message: str | None = None
    memory_usage: float | None = None
    start_time: datetime | None = None
    end_time: datetime | None = None


class PerformanceMonitor:
    """Performance monitoring utilities for tracking function execution metrics."""

    def __init__(self, logger: logging.Logger | None = None):
        """Initialize performance monitor.

        Args:
            logger: Logger instance. Uses default if None.
        """
        self.logger = logger or WorkerLogger.get_logger(__name__)
        self.metrics: dict[str, list] = {}

    def __call__(self, func: Callable) -> Callable:
        """Decorator to monitor function performance."""

        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            return self.monitor_execution(func, *args, **kwargs)

        return wrapper

    def monitor_execution(self, func: Callable, *args, **kwargs) -> Any:
        """Monitor function execution and collect metrics.

        Args:
            func: Function to monitor
            *args: Function arguments
            **kwargs: Function keyword arguments

        Returns:
            Function result
        """
        start_time = time.time()
        start_datetime = datetime.now(UTC)
        function_name = f"{func.__module__}.{func.__name__}"

        try:
            # Get memory usage before execution (if psutil available)
            memory_before = self._get_memory_usage()

            self.logger.debug(f"Starting execution of {function_name}")

            # Execute function
            result = func(*args, **kwargs)

            # Calculate metrics
            end_time = time.time()
            end_datetime = datetime.now(UTC)
            execution_time = end_time - start_time
            memory_after = self._get_memory_usage()
            memory_usage = (
                memory_after - memory_before if memory_before and memory_after else None
            )

            # Create metrics record
            metrics = PerformanceMetrics(
                function_name=function_name,
                execution_time=execution_time,
                success=True,
                memory_usage=memory_usage,
                start_time=start_datetime,
                end_time=end_datetime,
            )

            # Log successful execution
            self.logger.info(
                f"Function {function_name} completed successfully",
                extra={
                    "execution_time": execution_time,
                    "memory_usage": memory_usage,
                    "performance_metrics": asdict(metrics),
                },
            )

            # Store metrics
            self._store_metrics(metrics)

            return result

        except Exception as e:
            # Calculate metrics for failed execution
            end_time = time.time()
            end_datetime = datetime.now(UTC)
            execution_time = end_time - start_time

            metrics = PerformanceMetrics(
                function_name=function_name,
                execution_time=execution_time,
                success=False,
                error_type=type(e).__name__,
                error_message=str(e),
                start_time=start_datetime,
                end_time=end_datetime,
            )

            # Log failed execution
            self.logger.error(
                f"Function {function_name} failed after {execution_time:.3f}s",
                extra={
                    "execution_time": execution_time,
                    "error_type": type(e).__name__,
                    "error_message": str(e),
                    "performance_metrics": asdict(metrics),
                },
                exc_info=True,
            )

            # Store metrics
            self._store_metrics(metrics)

            raise

    def _get_memory_usage(self) -> float | None:
        """Get current memory usage in MB."""
        try:
            import psutil

            process = psutil.Process(os.getpid())
            return process.memory_info().rss / 1024 / 1024  # Convert to MB
        except ImportError:
            return None
        except Exception:
            return None

    def _store_metrics(self, metrics: PerformanceMetrics):
        """Store metrics for later analysis."""
        function_name = metrics.function_name
        if function_name not in self.metrics:
            self.metrics[function_name] = []

        self.metrics[function_name].append(metrics)

        # Keep only last 100 metrics per function to prevent memory bloat
        if len(self.metrics[function_name]) > 100:
            self.metrics[function_name] = self.metrics[function_name][-100:]

    def get_metrics(self, function_name: str | None = None) -> dict[str, Any]:
        """Get performance metrics.

        Args:
            function_name: Specific function name. Returns all if None.

        Returns:
            Metrics dictionary
        """
        if function_name:
            function_metrics = self.metrics.get(function_name, [])
        else:
            function_metrics = []
            for metrics_list in self.metrics.values():
                function_metrics.extend(metrics_list)

        if not function_metrics:
            return {}

        # Calculate aggregated metrics
        total_executions = len(function_metrics)
        successful_executions = sum(1 for m in function_metrics if m.success)
        failed_executions = total_executions - successful_executions

        execution_times = [m.execution_time for m in function_metrics]
        avg_execution_time = sum(execution_times) / len(execution_times)
        min_execution_time = min(execution_times)
        max_execution_time = max(execution_times)

        return {
            "function_name": function_name,
            "total_executions": total_executions,
            "successful_executions": successful_executions,
            "failed_executions": failed_executions,
            "success_rate": successful_executions / total_executions
            if total_executions > 0
            else 0,
            "avg_execution_time": avg_execution_time,
            "min_execution_time": min_execution_time,
            "max_execution_time": max_execution_time,
            "recent_metrics": [
                asdict(m) for m in function_metrics[-10:]
            ],  # Last 10 executions
        }

    def get_summary(self) -> dict[str, Any]:
        """Get summary of all monitored functions."""
        summary = {}
        for function_name in self.metrics:
            summary[function_name] = self.get_metrics(function_name)

        return summary


@contextmanager
def log_context(**kwargs):
    """Context manager for temporary logging context.

    Usage:
        with log_context(task_id='123', execution_id='456'):
            logger.info("This will include context")
    """
    # Save current context
    original_context = WorkerLogger.get_context()

    try:
        # Update context
        WorkerLogger.update_context(**kwargs)
        yield
    finally:
        # Restore original context
        if original_context:
            WorkerLogger.set_context(original_context)
        else:
            WorkerLogger.clear_context()


def with_execution_context(func: Callable) -> Callable:
    """Decorator to automatically set up logging context for execution functions.

    This decorator extracts common execution context parameters from function arguments
    and sets up logging context automatically, eliminating the need for repetitive
    log_context() usage in task functions.

    Expected function signature patterns:
    - func(self, schema_name, workflow_id, execution_id, pipeline_id=None, ...)
    - func(self, schema_name, execution_id, workflow_id, organization_id=None, ...)

    Args:
        func: The function to wrap with execution context

    Returns:
        Wrapped function with automatic logging context setup

    Usage:
        @with_execution_context
        def my_task(self, schema_name: str, workflow_id: str, execution_id: str, ...):
            # Function body - logging context is automatically available
            logger.info("This will include task_id, execution_id, etc.")
    """
    from functools import wraps

    @wraps(func)
    def wrapper(task_instance, *args, **kwargs) -> Any:
        # Extract task_id from Celery task instance
        task_id = (
            getattr(task_instance.request, "id", None)
            if hasattr(task_instance, "request")
            else None
        )

        # Extract common context parameters from arguments
        # Handle different argument patterns used across tasks
        context = {"task_id": task_id}

        if args:
            # Most common pattern: (schema_name, workflow_id, execution_id, ...)
            if len(args) >= 1:
                schema_name = args[0]
                context["organization_id"] = schema_name

            if len(args) >= 2:
                # Could be workflow_id or execution_id depending on task
                if len(args) >= 3:
                    # Pattern: (schema_name, workflow_id, execution_id, ...)
                    context["workflow_id"] = args[1]
                    context["execution_id"] = args[2]
                else:
                    # Pattern: (schema_name, execution_id, ...)
                    context["execution_id"] = args[1]

            if len(args) >= 4:
                # Check if 4th argument is pipeline_id or another param
                pipeline_id = (
                    args[3] if args[3] is not None else kwargs.get("pipeline_id")
                )
                if pipeline_id:
                    context["pipeline_id"] = pipeline_id

        # Extract additional context from kwargs
        for key in ["workflow_id", "execution_id", "pipeline_id", "organization_id"]:
            if key in kwargs and kwargs[key] is not None:
                context[key] = kwargs[key]

        # Remove None values from context
        context = {k: v for k, v in context.items() if v is not None}

        # Execute function with logging context
        with log_context(**context):
            return func(task_instance, *args, **kwargs)

    return wrapper


def logged_execution(logger: logging.Logger | None = None):
    """Decorator for automatic logging of function execution.

    Args:
        logger: Logger instance. Uses default if None.

    Returns:
        Decorated function
    """

    def decorator(func: Callable) -> Callable:
        actual_logger = logger or WorkerLogger.get_logger(func.__module__)

        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            function_name = f"{func.__module__}.{func.__name__}"
            actual_logger.debug(f"Starting execution of {function_name}")

            try:
                result = func(*args, **kwargs)
                actual_logger.debug(f"Successfully completed {function_name}")
                return result
            except Exception as e:
                actual_logger.error(
                    f"Function {function_name} failed: {e}", exc_info=True
                )
                raise

        return wrapper

    return decorator


# Create global performance monitor instance
performance_monitor = PerformanceMonitor()


# Convenience decorators
def monitor_performance(func: Callable) -> Callable:
    """Decorator for performance monitoring."""
    return performance_monitor(func)


def log_execution(func: Callable) -> Callable:
    """Decorator for execution logging."""
    return logged_execution()(func)


_REQUEST_ID_KEYS: tuple[str, ...] = (
    "request_id",
    "file_execution_id",
    "execution_id",
    "run_id",
)


def _coerce_id(value: Any) -> str | None:
    """Return a safe string id, or None if value is not a usable id type.

    Rejects arbitrary objects so a misnamed payload field (e.g. a dataclass
    or a list) cannot leak its ``__str__`` into log lines or OTel attributes.
    Booleans are excluded explicitly because ``bool`` is a subclass of
    ``int`` in Python; without the guard, ``True``/``False`` would
    serialize to the literal strings ``"True"``/``"False"``.
    """
    if isinstance(value, bool):
        return None
    if isinstance(value, str):
        return value or None
    if isinstance(value, (int, UUID)):
        return str(value)
    return None


def _bind_positional_args_to_names(
    args: Sequence[Any], kwargs: Mapping[str, Any], task: Any
) -> Mapping[str, Any] | None:
    """Map positional args to parameter names via the task's signature.

    Catches the case where ids are passed as positional strings (e.g.
    ``send_task("async_execute_bin", args=[schema, wf_id, exec_id, ...])``).
    Returns ``None`` if the task is not introspectable or has no resolved
    arguments.
    """
    if task is None:
        return None
    runner = getattr(task, "run", task)
    try:
        bound = inspect.signature(runner).bind_partial(*(args or ()), **(kwargs or {}))
    except (TypeError, ValueError):
        return None
    return dict(bound.arguments) if bound.arguments else None


def _arg_as_mapping(arg: Any) -> Mapping[str, Any] | None:
    """Return a mapping view of a positional arg, or None if not coercible.

    Accepts ``Mapping`` instances directly and dataclass instances via
    ``dataclasses.asdict``.
    """
    if isinstance(arg, Mapping):
        return arg
    if dataclasses.is_dataclass(arg) and not isinstance(arg, type):
        return dataclasses.asdict(arg)
    return None


def _gather_containers(
    args: Sequence[Any], kwargs: Mapping[str, Any], task: Any
) -> list[Mapping[str, Any]]:
    """Build the ordered list of mappings to scan for an id.

    Order (used only for stable iteration; final priority is by KEY in
    ``_extract_request_id``):

      1. Positional args bound to parameter names via the task signature.
      2. Top-level kwargs.
      3. ``Mapping`` values nested inside kwargs.
      4. ``Mapping`` / dataclass positional args.
    """
    containers: list[Mapping[str, Any]] = []

    bound = _bind_positional_args_to_names(args, kwargs, task)
    if bound:
        containers.append(bound)

    if kwargs:
        containers.append(kwargs)
        containers.extend(v for v in kwargs.values() if isinstance(v, Mapping))

    for arg in args or ():
        mapping = _arg_as_mapping(arg)
        if mapping is not None:
            containers.append(mapping)

    return containers


def _extract_request_id(
    args: Sequence[Any], kwargs: Mapping[str, Any], task: Any = None
) -> str | None:
    """Pull a usable request_id from a Celery task payload.

    Workers bind ``file_execution_id`` to the ``request_id`` log field
    (legacy structure-tool convention), giving per-file granularity in
    logs across the multi-tool execution chain.  Cross-service correlation
    across the API/HTTP boundary is handled by OpenTelemetry ``trace_id``,
    not by ``request_id``.

    The key priority below is also the migration path: callers may start
    passing a real HTTP ``request_id`` in the payload at any time and it
    will take precedence over ``file_execution_id`` automatically -- no
    worker change required.

    Priority is by KEY (not by container), so a payload with both
    ``request_id`` in one container and ``file_execution_id`` in another
    deterministically picks ``request_id`` regardless of insertion order.
    """
    containers = _gather_containers(args, kwargs, task)
    for key in _REQUEST_ID_KEYS:
        for container in containers:
            value = _coerce_id(container.get(key))
            if value:
                return value
    return None


def _bind_task_context(task_id, task, args, kwargs, **_):
    """Celery ``task_prerun`` handler: bind request_id onto the log context.

    Catches any extraction failure so a malformed payload can never leave
    the previous task's id bound on the thread.
    """
    try:
        request_id = _extract_request_id(args or (), kwargs or {}, task) or task_id
    except Exception:
        logging.getLogger(__name__).debug(
            "request_id extraction failed for task %s; falling back to task_id",
            task_id,
            exc_info=True,
        )
        request_id = task_id
    WorkerLogger.update_context(request_id=request_id, task_id=task_id)


def _clear_task_context(**_):
    """Celery ``task_postrun`` handler: reset task-scoped fields only.

    Preserves baseline context (``worker_name``, etc.) set at
    ``WorkerLogger.configure()``; only nulls out the per-task fields bound
    in ``_bind_task_context``.
    """
    WorkerLogger.update_context(request_id=None, task_id=None)


@functools.lru_cache(maxsize=1)
def _install_celery_request_id_signals() -> None:
    """Wire Celery signals once per process.

    Idempotent and thread-safe via ``functools.lru_cache``.  No-ops with a
    debug log if Celery is not importable (e.g. unit tests).
    """
    try:
        from celery.signals import task_postrun, task_prerun
    except ImportError as exc:
        logging.getLogger(__name__).debug(
            "celery.signals not importable; request_id signal install skipped: %s",
            exc,
        )
        return
    task_prerun.connect(_bind_task_context, weak=False)
    task_postrun.connect(_clear_task_context, weak=False)


# Dataclass/Dictionary Access Utilities


def safe_get_attr(obj: Any, key: str, default: Any = None) -> Any:
    """Safely get value from object using both attribute and dictionary access patterns.

    This utility handles the common pattern where objects may be either dataclass instances
    or dictionaries, providing consistent access regardless of the actual type.

    Args:
        obj: Object to get value from (dataclass instance or dictionary)
        key: Key/attribute name to access
        default: Default value if key/attribute not found

    Returns:
        Value from object or default if not found

    Examples:
        >>> safe_get_attr(WorkerFileData(execution_id="123"), "execution_id")
        "123"
        >>> safe_get_attr({"execution_id": "456"}, "execution_id")
        "456"
        >>> safe_get_attr({}, "missing_key", "default")
        "default"
    """
    # First try attribute access (for dataclass objects)
    if hasattr(obj, key):
        return getattr(obj, key, default)
    # Then try dictionary access
    elif isinstance(obj, dict):
        return obj.get(key, default)
    # Fallback to default
    else:
        return default


def ensure_dict_access(obj: Any, keys: list, default: Any = None) -> dict[str, Any]:
    """Extract multiple values from object using safe access patterns.

    Args:
        obj: Object to extract values from
        keys: List of keys/attributes to extract
        default: Default value for missing keys

    Returns:
        Dictionary with extracted values

    Example:
        >>> ensure_dict_access(file_data, ["execution_id", "workflow_id", "organization_id"])
        {'execution_id': '123', 'workflow_id': '456', 'organization_id': 'org_789'}
    """
    return {key: safe_get_attr(obj, key, default) for key in keys}