LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Log Consumer Tasks
This module contains Celery tasks for processing execution logs.
"""
import json
import os
from typing import Any
from urllib.parse import quote
import socketio
from queue_backend import worker_task
from shared.infrastructure.config import WorkerConfig
from shared.infrastructure.logging import WorkerLogger
from shared.utils.api_client_singleton import get_singleton_api_client
from unstract.core.cache.redis_client import build_socketio_redis_url
from unstract.core.cache.redis_queue_client import RedisQueueClient
from unstract.core.constants import LogEventArgument, LogProcessingTask
from unstract.core.log_utils import store_execution_log
logger = WorkerLogger.get_logger(__name__)
# Initialize worker configuration
config = WorkerConfig.from_env("LOG_CONSUMER")
# Redis configuration
redis_client = RedisQueueClient.from_env()
# Log storage configuration
log_queue_name = os.getenv("LOG_HISTORY_QUEUE_NAME", "log_history_queue")
log_storage_enabled = os.getenv("ENABLE_LOG_HISTORY", "true").lower() == "true"
# Socket.IO client for emitting events (uses same KombuManager as backend)
_sentinel_mode = os.getenv("REDIS_SENTINEL_MODE", "False").strip().lower() == "true"
_redis_host = os.getenv("REDIS_HOST", "localhost")
_redis_port = os.getenv("REDIS_PORT", "6379")
_redis_password = os.getenv("REDIS_PASSWORD", "")
_redis_user = os.getenv("REDIS_USER", "")
_cred_prefix = ""
if _redis_user and _redis_password:
_cred_prefix = f"{quote(_redis_user, safe='')}:{quote(_redis_password, safe='')}@"
elif _redis_password:
_cred_prefix = f":{quote(_redis_password, safe='')}@"
_transport_options: dict[str, Any] = {}
if _sentinel_mode:
_redis_db = os.getenv("REDIS_DB", "0") or "0"
socket_io_manager_url = (
f"sentinel://{_cred_prefix}{_redis_host}:{_redis_port}/{_redis_db}"
)
_sentinel_master_name = os.getenv("REDIS_SENTINEL_MASTER_NAME", "mymaster")
_transport_options = {"master_name": _sentinel_master_name}
else:
# UN-4123: built by unstract.core so this cannot drift from the backend.
# Hand-building it here would hardcode a scheme, and a publisher stuck on
# redis:// cannot reach a TLS-only endpoint — execution logs would simply stop
# arriving in the UI, with nothing failing loudly.
socket_io_manager_url = build_socketio_redis_url()
sio = socketio.Server(
async_mode="threading",
logger=False,
engineio_logger=False,
client_manager=socketio.KombuManager(
url=socket_io_manager_url,
write_only=True,
**(
{"connection_options": {"transport_options": _transport_options}}
if _transport_options
else {}
),
),
)
@worker_task(name=LogProcessingTask.TASK_NAME)
def logs_consumer(**kwargs: Any) -> None:
"""Task to process logs from log publisher.
This task processes execution logs by:
1. Storing them to Redis queue for persistence
2. Triggering WebSocket emission through backend API
Args:
kwargs: The arguments to process the logs.
Expected arguments:
USER_SESSION_ID: The room to be processed.
EVENT: The event to be processed Ex: logs:{session_id}.
MESSAGE: The message to be processed Ex: execution log.
"""
log_message = kwargs.get(LogEventArgument.MESSAGE)
room = kwargs.get(LogEventArgument.USER_SESSION_ID)
event = kwargs.get(LogEventArgument.EVENT)
logger.debug(f"[{os.getpid()}] Log message received: {log_message} for room {room}")
# Validate required arguments
if not room or not event:
logger.warning(f"Message received without room and event: {log_message}")
return
# Store execution log to Redis
try:
store_execution_log(
data=log_message,
redis_client=redis_client.redis_client,
log_queue_name=log_queue_name,
is_enabled=log_storage_enabled,
)
except Exception as e:
logger.error(f"Failed to store execution log: {e}")
# Emit WebSocket event directly through Socket.IO (via Redis broker)
try:
# Coerce UUID/datetime/etc. to JSON-safe primitives — Socket.IO's
# serializer is stdlib json and chokes on non-primitive types.
safe_message = json.loads(json.dumps(log_message, default=str))
payload = {"data": safe_message}
sio.emit(event, data=payload, room=room)
logger.debug(f"WebSocket event emitted successfully for room {room}")
except Exception as e:
logger.error(f"Failed to emit WebSocket event: {e}")
# Health check task for monitoring
@worker_task(name="log_consumer_health_check")
def health_check() -> dict[str, Any]:
"""Health check task for log consumer worker.
Returns:
Health status information
"""
try:
# Check Redis connectivity
redis_client.ping()
redis_status = "healthy"
except Exception as e:
redis_status = f"unhealthy: {e}"
try:
# Check API client connectivity
api_client = get_singleton_api_client(config)
api_status = "healthy" if api_client else "unhealthy"
except Exception as e:
api_status = f"unhealthy: {e}"
return {
"worker": "log_consumer",
"status": "healthy"
if redis_status == "healthy" and "healthy" in api_status
else "degraded",
"redis": redis_status,
"api": api_status,
"queue": log_queue_name,
"log_storage_enabled": log_storage_enabled,
}