Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

22.1 KB
Raw Blame
#!/usr/bin/env python
"""Unified Celery Worker Entry Point

This module serves as the main entry point for all Celery workers.
It uses WorkerBuilder to ensure proper configuration including chord retry settings.
"""

import importlib.util
import logging
import os
import sys
import threading
import time

# CRITICAL: Enable gRPC fork support BEFORE any Google API imports
# This must be set before importing any gRPC-based libraries (Google Cloud, etc.)
# to prevent SIGSEGV crashes in forked worker processes
os.environ.setdefault("GRPC_ENABLE_FORK_SUPPORT", "1")
os.environ.setdefault("GRPC_POLL_STRATEGY", "poll")

from celery import bootsteps, signals  # noqa: E402

# Add the workers directory to Python path
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))

# Import the WorkerBuilder and WorkerType
from shared.api.internal_client import InternalAPIClient  # noqa: E402
from shared.enums.worker_enums import WorkerType  # noqa: E402
from shared.infrastructure import initialize_worker_infrastructure  # noqa: E402
from shared.infrastructure.config.builder import WorkerBuilder  # noqa: E402
from shared.models.worker_models import get_celery_setting  # noqa: E402
from shared.patterns.factory.client_factory import ClientFactory  # noqa: E402

# Determine worker type from environment FIRST
WORKER_TYPE = os.environ.get("WORKER_TYPE", "general")

# Convert WORKER_TYPE string to WorkerType enum
# Build mapping dynamically from all available WorkerType enum values
# This automatically includes pluggable workers without hardcoding them
worker_type_mapping = {}
for wt in WorkerType:
    # Add both underscore and hyphen versions
    worker_type_mapping[wt.value] = wt
    worker_type_mapping[wt.value.replace("-", "_")] = wt
    worker_type_mapping[wt.value.replace("_", "-")] = wt

# Get the WorkerType enum
worker_type = worker_type_mapping.get(WORKER_TYPE, WorkerType.GENERAL)

# CRITICAL: Setup logging IMMEDIATELY before any logging calls
# This ensures ALL subsequent logs use Django format
WorkerBuilder.setup_logging(worker_type)

# Now get logger after setup is complete
logger = logging.getLogger(__name__)


# ============= ADD HEARTBEATKEEPER CLASS HERE =============
class HeartbeatKeeper(bootsteps.StartStopStep):
    """Maintains RabbitMQ heartbeat ONLY during warm shutdown to prevent task duplication

    During normal operation: Remains dormant - Celery's built-in heartbeat thread handles everything
    During warm shutdown: Takes over heartbeat duties when Celery's thread stops

    This fixes the issue where RabbitMQ requeues tasks during K8s pod termination
    because Celery stops sending heartbeats during warm shutdown.
    """

    requires = {"celery.worker.components:Pool"}

    def __init__(self, worker, worker_type, **kwargs):
        self.worker = worker
        self.worker_type = worker_type
        self.heartbeat_thread = None
        self.should_stop = threading.Event()
        self._shutdown_in_progress = False

    def start(self, worker):
        """Start heartbeat keeper thread - remains dormant until warm shutdown"""
        # Load all configuration once and store as instance variables
        # Read broker_heartbeat from actual Celery configuration (single source of truth)
        # Default to 10 seconds if None (Celery's default is None)
        self.broker_heartbeat = self.worker.app.conf.broker_heartbeat or 10

        # Get HeartbeatKeeper-specific configuration using hierarchical config system
        self.shutdown_interval = get_celery_setting(
            "HEARTBEAT_KEEPER_SHUTDOWN_INTERVAL",
            self.worker_type,
            max(1, self.broker_heartbeat // 5),
            int,
        )
        self.normal_check_interval = get_celery_setting(
            "HEARTBEAT_KEEPER_CHECK_INTERVAL",
            self.worker_type,
            self.broker_heartbeat,
            int,
        )
        self.max_shutdown_errors = get_celery_setting(
            "HEARTBEAT_KEEPER_MAX_ERRORS", self.worker_type, 10, int
        )

        logger.info(
            "HeartbeatKeeper: Starting in standby mode (prevents task duplication during shutdown)"
        )
        logger.info(
            "HeartbeatKeeper: Will remain dormant during normal operation - Celery handles heartbeats"
        )
        logger.info(
            f"HeartbeatKeeper: Will activate during warm shutdown (interval: {self.shutdown_interval}s, broker timeout: {self.broker_heartbeat}s)"
        )

        self.heartbeat_thread = threading.Thread(
            target=self._heartbeat_loop,
            name="HeartbeatKeeper",
            daemon=True,  # Daemon thread won't block shutdown
        )
        self.heartbeat_thread.start()

    def stop(self, worker):
        """Called during warm shutdown - keep heartbeat alive"""
        self._shutdown_in_progress = True
        logger.warning(
            "HeartbeatKeeper: Warm shutdown detected - maintaining heartbeat for task completion"
        )
        # Let heartbeat continue during shutdown
        # K8s terminationGracePeriodSeconds handles the timeout

    def terminate(self, worker):
        """Called during final termination"""
        logger.info("HeartbeatKeeper: Final termination - stopping heartbeat thread")
        self.should_stop.set()
        if self.heartbeat_thread and self.heartbeat_thread.is_alive():
            self.heartbeat_thread.join(timeout=1)  # Short timeout since daemon thread

    def _is_connection_ready(self) -> tuple[bool, any]:
        """Check if worker connection is available and ready for heartbeat.

        Returns:
            tuple: (is_ready, consumer_or_none)
        """
        if not hasattr(self.worker, "consumer") or not self.worker.consumer:
            return (False, None)

        consumer = self.worker.consumer
        if not consumer.connection or not consumer.connection.connected:
            return (False, None)

        return (True, consumer)

    def _send_heartbeat_with_error_handling(
        self, consumer: any, consecutive_errors: int, last_heartbeat: float
    ) -> tuple[int, float]:
        """Send heartbeat and handle errors.

        Args:
            consumer: Celery consumer instance
            consecutive_errors: Current count of consecutive errors
            last_heartbeat: Timestamp of last successful heartbeat

        Returns:
            tuple: (updated_consecutive_errors, updated_last_heartbeat)
        """
        current_time = time.time()
        elapsed = current_time - last_heartbeat

        try:
            # Use heartbeat_check with rate=0 to force immediate send without validation
            consumer.connection.heartbeat_check(rate=0)
            logger.info(
                f"HeartbeatKeeper: Heartbeat sent during shutdown (elapsed: {elapsed:.1f}s)"
            )
            return (0, current_time)  # Reset errors on success, update timestamp

        except Exception as hb_error:
            consecutive_errors += 1
            # During shutdown, we expect some errors - log but continue
            if "Too many heartbeats missed" in str(hb_error):
                logger.warning(
                    f"HeartbeatKeeper: Connection degraded during shutdown "
                    f"(attempt {consecutive_errors}), continuing..."
                )
            else:
                logger.warning(
                    f"HeartbeatKeeper: Heartbeat error during shutdown: {hb_error}"
                )

            # Don't break the loop during shutdown - keep trying
            if consecutive_errors > self.max_shutdown_errors:
                logger.info(
                    "HeartbeatKeeper: Connection appears closed, but will continue attempts"
                )

            return (consecutive_errors, last_heartbeat)  # Keep old timestamp on error

    def _attempt_shutdown_heartbeat(
        self, consecutive_errors: int, last_heartbeat: float
    ) -> tuple[int, float]:
        """Attempt to send heartbeat during shutdown with full error handling.

        Args:
            consecutive_errors: Current count of consecutive errors
            last_heartbeat: Timestamp of last successful heartbeat

        Returns:
            tuple: (updated_consecutive_errors, updated_last_heartbeat)
        """
        try:
            is_ready, consumer = self._is_connection_ready()
            if not is_ready:
                return (consecutive_errors, last_heartbeat)

            return self._send_heartbeat_with_error_handling(
                consumer, consecutive_errors, last_heartbeat
            )

        except OSError as conn_error:
            logger.info(
                f"HeartbeatKeeper: Expected connection issue during shutdown: {conn_error}"
            )
        except Exception as e:
            logger.debug(f"HeartbeatKeeper: Error during shutdown (continuing): {e}")

        return (consecutive_errors, last_heartbeat)

    def _wait_in_standby_mode(self, logged_standby: bool) -> tuple[bool, bool]:
        """Handle standby mode during normal operation.

        Args:
            logged_standby: Whether we've already logged standby status

        Returns:
            tuple: (should_continue_loop, updated_logged_standby)
        """
        if not logged_standby:
            logger.info(
                "HeartbeatKeeper: In standby mode - Celery's heartbeat thread is active"
            )
            logged_standby = True

        # Wait and check periodically if shutdown has started
        self.should_stop.wait(self.normal_check_interval)
        return (True, logged_standby)

    def _handle_shutdown_transition(
        self, logged_standby: bool, last_heartbeat: float
    ) -> tuple[bool, float]:
        """Handle transition from standby to active shutdown mode.

        Args:
            logged_standby: Whether we're currently in logged standby state
            last_heartbeat: Current last heartbeat timestamp

        Returns:
            tuple: (updated_logged_standby, updated_last_heartbeat)
        """
        if logged_standby:
            logger.warning(
                "HeartbeatKeeper: Taking over heartbeat duties from Celery during warm shutdown"
            )
            logged_standby = False
            last_heartbeat = time.time()  # Reset timer when we start

        return (logged_standby, last_heartbeat)

    def _heartbeat_loop(self):
        """Maintain heartbeats ONLY during warm shutdown - dormant during normal operation"""
        # Initialize state variables
        consecutive_shutdown_errors = 0
        last_heartbeat = time.time()
        logged_standby = False

        # Log the configuration on startup (debug level to avoid noise)
        logger.debug(
            f"HeartbeatKeeper config: shutdown_interval={self.shutdown_interval}s, "
            f"check_interval={self.normal_check_interval}s, "
            f"broker_heartbeat={self.broker_heartbeat}s, "
            f"max_errors={self.max_shutdown_errors}"
        )

        while not self.should_stop.is_set():
            # During normal operation - just sleep, let Celery handle heartbeats
            if not self._shutdown_in_progress:
                should_continue, logged_standby = self._wait_in_standby_mode(
                    logged_standby
                )
                if should_continue:
                    continue

            # Handle transition from standby to active shutdown mode
            logged_standby, last_heartbeat = self._handle_shutdown_transition(
                logged_standby, last_heartbeat
            )

            # Active heartbeat sending during shutdown
            (
                consecutive_shutdown_errors,
                last_heartbeat,
            ) = self._attempt_shutdown_heartbeat(
                consecutive_shutdown_errors, last_heartbeat
            )

            # Short wait during shutdown for frequent heartbeats
            self.should_stop.wait(self.shutdown_interval)


# ============= END OF HEARTBEATKEEPER CLASS =============


# ============= WORKER-TYPE-AWARE HEARTBEATKEEPER =============
class WorkerTypeAwareHeartbeatKeeper(HeartbeatKeeper):
    """HeartbeatKeeper with worker_type from module context.

    This wrapper class captures the worker_type from the module-level variable
    and passes it to HeartbeatKeeper's __init__. This allows Celery's bootstep
    system to instantiate it with just (worker, **kwargs) while still providing
    the worker_type needed for configuration.
    """

    def __init__(self, worker, **kwargs):
        # worker_type is captured from module scope via closure
        super().__init__(worker, worker_type, **kwargs)


# ============= END OF WORKER-TYPE-AWARE HEARTBEATKEEPER =============


logger.info("🚀 Unified Worker Entry Point - Using WorkerBuilder System")
logger.info(f"📋 Worker Type: {WORKER_TYPE}")
logger.info(f"🐳 Running from: {os.getcwd()}")
logger.info(f"📦 Converted '{WORKER_TYPE}' to {worker_type}")

# Use WorkerBuilder to create the Celery app with proper configuration
# This ensures chord retry configuration is applied correctly
logger.info(f"🔧 Building Celery app using WorkerBuilder for {worker_type}")
app, config = WorkerBuilder.build_celery_app(worker_type)


# ============= WORKER PROCESS INIT HOOK FOR GRPC FORK-SAFETY =============
@signals.worker_process_init.connect
def on_worker_process_init(**kwargs):
    """Initialize worker process after fork.

    This signal handler runs in each forked worker process AFTER the fork.

    1. SIGTERM immunity: Ignore SIGTERM in child workers so the MainProcess
       can orchestrate graceful shutdown via pool.close()/pool.join().
       Without this, container runtimes (e.g. containerd 2.0) that send
       SIGTERM to all processes in the cgroup kill children immediately,
       causing WorkerLostError and orphaned tool containers.
       billiard's reset_signals() explicitly preserves SIG_IGN, so this
       survives into task execution. SIGUSR1 (soft timeout) and SIGKILL
       (hard timeout) are unaffected.

    2. gRPC fork-safety: Ensure gRPC connections and Google API clients are
       created fresh in the child process, not inherited from the parent.
       Without this, Google Cloud libraries (Drive, GCS, BigQuery, Secret
       Manager) crash with SIGSEGV because they inherit stale gRPC
       connections from the parent process.
    """
    import gc
    import signal

    # Ignore SIGTERM in child workers - let MainProcess handle shutdown
    signal.signal(signal.SIGTERM, signal.SIG_IGN)

    logger.info(
        "Worker process initialized after fork (PID: %s) - SIGTERM ignored, "
        "gRPC fork support enabled",
        os.getpid(),
    )

    # Force garbage collection to clean up any stale gRPC state from parent
    gc.collect()


# ============= END OF WORKER PROCESS INIT HOOK =============


# ============= WORKER PROCESS SHUTDOWN HOOK =============
@signals.worker_process_shutdown.connect
def on_worker_process_shutdown(**kwargs):
    """Clean up HTTP sessions during worker shutdown."""
    logger.info("Cleaning up API client resources (PID: %s)", os.getpid())
    try:
        InternalAPIClient.reset_singleton()
    except Exception as e:
        logger.warning(f"Failed to reset InternalAPIClient singleton: {e}")
    try:
        ClientFactory.reset_shared_state()
    except Exception as e:
        logger.warning(f"Failed to reset ClientFactory: {e}")


# ============= END OF WORKER PROCESS SHUTDOWN HOOK =============


# ============= TASK COMPLETION HOOK (SESSION LIFECYCLE SAFETY VALVE) =============
@signals.task_postrun.connect
def on_task_postrun(sender=None, task_id=None, **kwargs):
    """Increment task counter for periodic singleton session reset (FR-3.2).

    After WORKER_SINGLETON_RESET_THRESHOLD tasks (default: 1000), the singleton
    session is closed and recreated. Skipped when singleton is disabled (default)
    since the counter and reset would be no-ops.
    """
    if not config.enable_api_client_singleton:
        return
    try:
        InternalAPIClient.increment_task_counter()
    except Exception as e:
        logger.warning(f"Failed to increment task counter: {e}")


# ============= END OF TASK COMPLETION HOOK =============


# ============= REGISTER HEARTBEATKEEPER HERE =============
# Check if HeartbeatKeeper should be enabled (default: enabled)
# Uses hierarchical config system: CLI args > worker-specific > global > default
heartbeat_keeper_enabled = get_celery_setting(
    "HEARTBEAT_KEEPER_ENABLED", worker_type, True, bool
)

if heartbeat_keeper_enabled:
    # Register the HeartbeatKeeper bootstep (class defined at module level)
    app.steps["worker"].add(WorkerTypeAwareHeartbeatKeeper)
    logger.info(
        "✅ HeartbeatKeeper registered (dormant until warm shutdown) to prevent task duplication"
    )
else:
    logger.info(
        "⚠️ HeartbeatKeeper DISABLED - tasks may be duplicated if worker is terminated during warm shutdown"
    )
# ============= END OF REGISTRATION =============


# Initialize worker infrastructure (singleton API clients, cache managers, etc.)
# This must happen BEFORE task imports so tasks can use shared infrastructure
logger.info("🏗️ Initializing worker infrastructure (singleton pattern)...")

initialize_worker_infrastructure()
logger.info("✅ Worker infrastructure initialized successfully")


def load_worker_tasks(worker_type: WorkerType) -> None:
    """Register the worker type's Celery tasks.

    For pluggable workers, ``build_celery_app()`` above already imported the
    plugin package via ``WorkerBuilder._verify_pluggable_worker_exists``
    (``importlib.import_module("pluggable_worker.<type>.worker")`` — a proper
    PACKAGE import that RUNS the plugin's own registration code, e.g.
    ``from . import tasks`` with relative imports such as
    ``from .clients import ...``). That registration's ``@shared_task`` bindings
    are applied when the app FINALIZES — which any read of ``app.tasks`` triggers
    automatically (``finalize(auto=True)``), so they are in place by the time
    anything inspects the registry. The generic file-path load below is SKIPPED for
    pluggable workers because loading ``tasks.py`` under a bare ``"tasks"`` spec (no
    parent package) would break the plugin's relative imports ("attempted relative
    import with no known parent package") — not because of any binding-order
    subtlety.

    Non-pluggable (top-level) workers use absolute imports; their ``tasks.py`` is
    loaded by file path with the worker directory on ``sys.path``. Their tasks bind
    eagerly here, so a missing directory / ``tasks.py`` is a broken deploy and
    hard-fails rather than booting a no-op worker.
    """
    if worker_type.is_pluggable():
        logger.info(
            f"✅ Pluggable worker {worker_type.value} tasks registered via "
            "WorkerBuilder (package import); skipping file-path task load"
        )
        return

    base_dir = os.path.dirname(os.path.abspath(__file__))
    worker_directory = worker_type.to_directory()
    worker_path = os.path.join(base_dir, worker_directory)
    if not os.path.exists(worker_path):
        # Non-pluggable only (pluggable returned above): tasks bind eagerly here
        # with no later finalize step, so a missing worker dir is terminal.
        raise RuntimeError(f"Worker directory not found: {worker_path}")

    sys.path.append(worker_path)
    logger.info(f"✅ Added {worker_directory} to Python path for task imports")

    tasks_file = os.path.join(worker_path, "tasks.py")
    if not os.path.exists(tasks_file):
        raise RuntimeError(f"No tasks.py found for worker at: {tasks_file}")

    logger.info(f"📋 Loading tasks from: {tasks_file}")
    spec = importlib.util.spec_from_file_location("tasks", tasks_file)
    tasks_module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(tasks_module)
    logger.info(f"✅ Tasks loaded successfully from {worker_directory}")


load_worker_tasks(worker_type)

# A worker that boots with no registered tasks starts but silently processes
# nothing (Celery does not error on an empty registry). Surface that misconfig.
#
# NB: reading ``app.tasks`` below AUTO-FINALIZES the app (Celery's ``tasks``
# property calls ``self.finalize(auto=True)``), so every pending ``@shared_task`` /
# ``connect_on_app_finalize`` registration has already been applied by the time the
# emptiness test is evaluated. The check is therefore exactly as accurate for
# pluggable workers as for non-pluggable ones — an earlier version of this comment
# claimed the opposite ("bind on finalize, which can be after this point") and used
# it to justify the split; that premise was wrong. Two real consequences: the app is
# finalized at import rather than at worker start, and an empty registry here is a
# genuine misconfiguration for BOTH kinds.
#
# The split is kept deliberately, on deployment-risk grounds rather than accuracy:
# a non-pluggable worker's tasks are bound eagerly by ``load_worker_tasks`` in this
# same module, so an empty registry is unambiguously terminal — RAISE. Pluggable
# workers register via plugin package import, whose availability is deployment
# dependent; hard-failing there would turn one absent/misconfigured plugin into a
# container crash-loop across the fleet, so it stays a loud WARN. Tightening this to
# a raise is worthwhile but needs a pass confirming every deployed pluggable worker
# registers at import time — tracked under UN-3445 hardening, not changed blind here.
if not any(not name.startswith("celery.") for name in app.tasks):
    _empty_registry_msg = (
        f"No non-celery tasks registered for worker '{worker_type.value}' "
        f"(pluggable={worker_type.is_pluggable()})."
    )
    if worker_type.is_pluggable():
        logger.warning(
            f"⚠️ {_empty_registry_msg} If this persists past app finalize the "
            "worker will start but process nothing — check the plugin task "
            "registration."
        )
    else:
        raise RuntimeError(
            f"{_empty_registry_msg} The worker would start but process nothing — "
            "check the worker task registration."
        )

# Log successful configuration
logger.info(f"✅ Successfully loaded {worker_type} worker using WorkerBuilder")
logger.info(
    f"📊 Chord retry interval: {getattr(app.conf, 'result_chord_retry_interval', 'NOT SET')}"
)
logger.info(f"🎯 Worker '{config.worker_name}' ready for Celery")

# Export for Celery to use
__all__ = ["app", "config"]