LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Worker Workflow Logger - WebSocket Log Publisher for Workers
This module provides WebSocket logging functionality for workers, matching
the backend's workflow_log.py functionality but adapted for the worker environment.
Usage:
# Initialize logger
workflow_logger = WorkerWorkflowLogger(
execution_id=execution_id,
log_stage=LogStage.PROCESSING,
file_execution_id=file_execution_id,
organization_id=organization_id
)
# Send logs to UI via WebSocket
workflow_logger.log_info(logger, "Processing file batch...")
workflow_logger.log_error(logger, "Failed to process file")
workflow_logger.publish_update_log(LogState.RUNNING, "File processing started")
"""
import logging
from typing import TYPE_CHECKING
from unstract.core.pubsub_helper import LogPublisher
from unstract.workflow_execution.enums import (
LogComponent,
LogLevel,
LogStage,
LogState,
)
from .logger import WorkerLogger
if TYPE_CHECKING:
from shared.api import InternalAPIClient
# Get worker logger for internal logging
logger = WorkerLogger.get_logger(__name__)
class WorkerWorkflowLogger:
"""Worker-compatible workflow logger that sends logs to UI via WebSocket.
This mirrors the backend's WorkflowLog class but adapted for workers:
- Uses worker StateStore equivalent
- Compatible with worker environment variables
- Maintains the same API for easy migration
"""
def __init__(
self,
execution_id: str,
log_stage: LogStage,
file_execution_id: str | None = None,
organization_id: str | None = None,
pipeline_id: str | None = None,
log_events_id: str | None = None,
):
"""Initialize workflow logger for worker environment.
Args:
execution_id: Workflow execution ID
log_stage: Current processing stage (SOURCE, DESTINATION, PROCESSING, etc.)
file_execution_id: Optional file execution ID for file-specific logs
organization_id: Organization ID for message routing
pipeline_id: Pipeline ID for message routing (fallback)
log_events_id: Explicit log events ID (overrides other channel logic)
"""
# Try to get log_events_id from worker StateStore if not provided
if not log_events_id:
try:
# Try to import and use the worker's StateStore equivalent
from ..context import StateStore
log_events_id = StateStore.get("LOG_EVENTS_ID")
except ImportError:
logger.debug(
"StateStore not available in worker, using pipeline_id for messaging channel"
)
# Determine messaging channel (matches backend logic)
self.messaging_channel = log_events_id if log_events_id else pipeline_id
self.execution_id = str(execution_id)
self.file_execution_id = str(file_execution_id) if file_execution_id else None
self.organization_id = str(organization_id) if organization_id else None
self.log_stage = log_stage
if not self.messaging_channel:
logger.warning(
f"No messaging channel available for execution {execution_id}. "
f"WebSocket logs may not be delivered to UI."
)
def publish_log(
self,
message: str,
level: LogLevel = LogLevel.INFO,
step: int | None = None,
cost_type: str | None = None,
cost_units: str | None = None,
cost_value: float | None = None,
) -> None:
"""Publish a log message to the WebSocket channel.
Args:
message: Log message to display in UI
level: Log level (INFO, ERROR)
step: Optional step number for multi-step processes
cost_type: Optional cost tracking type
cost_units: Optional cost units
cost_value: Optional cost value
"""
try:
if not self.messaging_channel:
logger.warning(f"No messaging channel, skipping WebSocket log: {message}")
return
log_details = LogPublisher.log_workflow(
stage=self.log_stage.value,
message=message,
level=level.value,
step=step,
execution_id=self.execution_id,
file_execution_id=self.file_execution_id,
organization_id=self.organization_id,
cost_type=cost_type,
cost_units=cost_units,
cost_value=cost_value,
)
# Publish to WebSocket channel
success = LogPublisher.publish(self.messaging_channel, log_details)
if not success:
logger.warning(f"Failed to publish WebSocket log: {message}")
except Exception as e:
logger.error(f"Error publishing WebSocket log: {e}")
def log_error(self, worker_logger: logging.Logger, message: str, **kwargs) -> None:
"""Log an error message both to worker logs and WebSocket.
Args:
worker_logger: Worker logger instance for console/file logging
message: Error message to log
**kwargs: Additional logging kwargs
"""
# Send to WebSocket for UI display
self.publish_log(message, level=LogLevel.ERROR)
# Log to worker logger for debugging
worker_logger.error(message, **kwargs)
def log_info(self, worker_logger: logging.Logger, message: str, **kwargs) -> None:
"""Log an info message both to worker logs and WebSocket.
Args:
worker_logger: Worker logger instance for console/file logging
message: Info message to log
**kwargs: Additional logging kwargs
"""
# Send to WebSocket for UI display
self.publish_log(message, level=LogLevel.INFO)
# Log to worker logger for debugging
worker_logger.info(message, **kwargs)
def publish_update_log(
self,
state: LogState,
message: str,
component: str | LogComponent | None = None,
) -> None:
"""Publish update logs for monitoring workflow execution progress.
These are used for status updates, progress indicators, and component state changes.
Args:
state: Log state (RUNNING, SUCCESS, ERROR, etc.)
message: Update message
component: Optional component name (SOURCE, DESTINATION, etc.)
"""
try:
if not self.messaging_channel:
logger.warning(
f"No messaging channel, skipping WebSocket update: {message}"
)
return
if isinstance(component, LogComponent):
component = component.value
log_details = LogPublisher.log_workflow_update(
state=state.value,
message=message,
component=component,
)
# Publish to WebSocket channel
success = LogPublisher.publish(self.messaging_channel, log_details)
if not success:
logger.warning(f"Failed to publish WebSocket update: {message}")
except Exception as e:
logger.error(f"Error publishing WebSocket update: {e}")
def publish_initial_workflow_logs(self, total_files: int) -> None:
"""Publish initial workflow startup logs.
Args:
total_files: Total number of files to process
"""
self.publish_update_log(
LogState.BEGIN_WORKFLOW,
f"Starting workflow execution with {total_files} files",
LogComponent.WORKFLOW,
)
def publish_source_logs(self, files_found: int, files_filtered: int = 0) -> None:
"""Publish source connector logs.
Args:
files_found: Number of files found by source connector
files_filtered: Number of files filtered out (optional)
"""
if files_filtered > 0:
message = f"Found {files_found} files, filtered to {files_found - files_filtered} files"
else:
message = f"Found {files_found} files from source"
self.publish_update_log(LogState.SUCCESS, message, LogComponent.SOURCE)
def publish_processing_start(
self, file_name: str, current_idx: int, total_files: int
) -> None:
"""Publish file processing start log.
Args:
file_name: Name of file being processed
current_idx: Current file index (1-based)
total_files: Total number of files
"""
message = f"Processing file {current_idx}/{total_files}: {file_name}"
self.publish_log(message, LogLevel.INFO)
def publish_processing_complete(
self, file_name: str, success: bool, error: str | None = None
) -> None:
"""Publish file processing completion log.
Args:
file_name: Name of processed file
success: Whether processing was successful
error: Error message if processing failed
"""
if success:
message = f"✓ File '{file_name}' processed successfully"
level = LogLevel.INFO
else:
message = f"✗ File '{file_name}' processing failed"
if error:
message += f" - {error}"
level = LogLevel.ERROR
self.publish_log(message, level)
def publish_destination_logs(
self, files_sent: int, destination_type: str = "destination"
) -> None:
"""Publish destination connector logs.
Args:
files_sent: Number of files sent to destination
destination_type: Type of destination (destination, manual_review, etc.)
"""
message = f"Sent {files_sent} files to {destination_type}"
self.publish_update_log(LogState.SUCCESS, message, LogComponent.DESTINATION)
def publish_execution_complete(
self, successful_files: int, failed_files: int, total_time: float
) -> None:
"""Publish execution completion summary.
Args:
successful_files: Number of successfully processed files
failed_files: Number of failed files
total_time: Total execution time in seconds
"""
if failed_files > 0:
message = f"Execution completed: {successful_files} successful, {failed_files} failed ({total_time:.1f}s)"
state = LogState.ERROR if successful_files == 0 else LogState.SUCCESS
else:
message = f"Execution completed successfully: {successful_files} files processed ({total_time:.1f}s)"
state = LogState.SUCCESS
self.publish_update_log(state, message, LogComponent.WORKFLOW)
@classmethod
def create_for_file_processing(
cls,
execution_id: str,
file_execution_id: str | None = None,
organization_id: str | None = None,
pipeline_id: str | None = None,
) -> "WorkerWorkflowLogger":
"""Factory method for file processing workflows.
Args:
execution_id: Workflow execution ID
file_execution_id: File execution ID
organization_id: Organization ID
pipeline_id: Pipeline ID
Returns:
Configured WorkerWorkflowLogger for file processing
"""
return cls(
execution_id=execution_id,
log_stage=LogStage.PROCESSING,
file_execution_id=file_execution_id,
organization_id=organization_id,
pipeline_id=pipeline_id,
)
@classmethod
def create_for_source(
cls,
execution_id: str,
organization_id: str | None = None,
pipeline_id: str | None = None,
) -> "WorkerWorkflowLogger":
"""Factory method for source connector workflows.
Args:
execution_id: Workflow execution ID
organization_id: Organization ID
pipeline_id: Pipeline ID
Returns:
Configured WorkerWorkflowLogger for source processing
"""
return cls(
execution_id=execution_id,
log_stage=LogStage.SOURCE,
organization_id=organization_id,
pipeline_id=pipeline_id,
)
@classmethod
def create_for_general_workflow(
cls,
execution_id: str,
organization_id: str | None = None,
pipeline_id: str | None = None,
log_events_id: str | None = None,
) -> "WorkerWorkflowLogger":
"""Factory method for general workflow execution.
Args:
execution_id: Workflow execution ID
organization_id: Organization ID
pipeline_id: Pipeline ID
log_events_id: Log events ID
Returns:
Configured WorkerWorkflowLogger for general workflow
"""
return cls(
execution_id=execution_id,
log_stage=LogStage.RUN,
organization_id=organization_id,
pipeline_id=pipeline_id,
log_events_id=log_events_id,
)
@classmethod
def create_for_api_workflow(
cls,
execution_id: str,
organization_id: str | None = None,
pipeline_id: str | None = None,
log_events_id: str | None = None,
) -> "WorkerWorkflowLogger":
"""Factory method for API workflow execution.
Args:
execution_id: Workflow execution ID
organization_id: Organization ID
pipeline_id: Pipeline ID (for API deployments)
log_events_id: Log events ID
Returns:
Configured WorkerWorkflowLogger for API workflow
"""
return cls(
execution_id=execution_id,
log_stage=LogStage.RUN, # Same as general workflow
organization_id=organization_id,
pipeline_id=pipeline_id,
log_events_id=log_events_id,
)
def create_file_logger(self, file_execution_id: str) -> "WorkerWorkflowLogger":
"""Create a file-specific logger from an existing workflow logger.
This preserves all the same settings but adds the file_execution_id for
file-specific log tracking.
Args:
file_execution_id: WorkflowFileExecution ID to associate with logs
Returns:
New WorkerWorkflowLogger instance with file_execution_id
"""
return WorkerWorkflowLogger(
execution_id=self.execution_id,
log_stage=self.log_stage,
file_execution_id=file_execution_id, # Add file association
organization_id=self.organization_id,
pipeline_id=None, # Use the messaging channel from parent
log_events_id=self.messaging_channel, # Preserve messaging channel
)
def publish_average_cost_log(
self,
worker_logger: logging.Logger,
total_files: int,
execution_id: str,
total_cost: float | None,
) -> None:
"""Publish average cost log for the workflow execution.
This matches the backend's WorkflowLog.publish_average_cost_log method
to provide consistent UI feedback for cost tracking.
Args:
worker_logger: Worker logger instance for console/file logging
total_files: Total number of files processed
execution_id: Execution ID for cost calculation
total_cost: Total aggregated cost in dollars (can be None)
"""
try:
if total_cost is not None and total_files > 0:
average_cost = round(total_cost / total_files, 5)
message = (
f"The average cost per file for execution "
f"'{execution_id}' is '${average_cost}'"
)
# Send to WebSocket for UI display
self.publish_log(message, level=LogLevel.INFO)
# Log to worker logger for debugging
worker_logger.info(message)
else:
# Handle cases where cost data is unavailable
if total_cost is None:
worker_logger.debug(
f"No cost data available for execution {execution_id}"
)
elif total_files == 0:
worker_logger.debug(f"No files processed in execution {execution_id}")
except Exception as e:
error_msg = f"Unable to get aggregated cost for '{execution_id}': {str(e)}"
worker_logger.warning(error_msg)
def publish_final_workflow_logs(
self,
total_files: int,
successful_files: int,
failed_files: int,
) -> None:
"""Publish final workflow execution logs.
This matches the backend's WorkflowLog.publish_final_workflow_logs method
to provide consistent UI feedback for execution completion.
Args:
total_files: Total number of files processed
successful_files: Number of successfully processed files
failed_files: Number of files that failed processing
"""
# Publish workflow end status to UI status bar
self.publish_update_log(LogState.END_WORKFLOW, "1", LogComponent.STATUS_BAR)
# Publish workflow completion status
self.publish_update_log(
LogState.SUCCESS, "Executed successfully", LogComponent.WORKFLOW
)
# Publish detailed execution summary
summary_message = (
f"Total files: {total_files}, "
f"{successful_files} successfully executed and {failed_files} error(s)"
)
self.publish_log(summary_message, level=LogLevel.INFO)
def log_total_cost_per_file(
self,
worker_logger: logging.Logger,
file_execution_id: str,
file_name: str,
api_client: "InternalAPIClient",
) -> None:
"""Log total cost for a specific file execution.
This matches the backend's WorkflowLog.log_total_cost_per_file method
which uses UsageHelper.get_aggregated_token_count() directly.
Args:
worker_logger: Worker logger instance for console/file logging
file_execution_id: File execution ID to get cost data for (run_id)
file_name: Name of the file being executed
api_client: Internal API client for usage data retrieval
"""
try:
# Use existing usage_client from InternalAPIClient (matches backend's UsageHelper.get_aggregated_token_count)
usage_response = api_client.usage_client.get_aggregated_token_count(
file_execution_id=file_execution_id,
organization_id=api_client.organization_id,
)
if not usage_response.is_success() or not usage_response.data:
# Handle no cost data - matches backend behavior (uses WARN level)
message = f"No cost data available for file '{file_name}'"
self.publish_log(message, level=LogLevel.ERROR)
worker_logger.warning(message)
return
# Extract cost from usage data (matches backend: cost_dict.get("cost_in_dollars"))
cost_in_dollars = usage_response.data.cost_in_dollars
if cost_in_dollars is None:
message = f"No cost data available for file '{file_name}'"
self.publish_log(message, level=LogLevel.ERROR)
worker_logger.warning(message)
return
# Round cost to 5 decimal places (matches backend implementation)
cost = round(cost_in_dollars or 0, 5)
# Log the total cost for the file (matches backend message format exactly)
message = f"Total cost for file '{file_name}' is '${cost}'"
# Send to WebSocket for UI display (matches backend: self.publish_log)
self.publish_log(message, level=LogLevel.INFO)
# Log to worker logger for debugging
worker_logger.info(message)
except Exception as e:
error_msg = f"Unable to get cost data for file '{file_name}' (execution: {file_execution_id}): {str(e)}"
# Send error to WebSocket for UI display
self.publish_log(error_msg, level=LogLevel.ERROR)
# Log to worker logger for debugging
worker_logger.warning(error_msg)