Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

7.4 KB
Raw Blame
"""Worker Type and Queue Name Enumerations

This module provides centralized enums for worker types and queue names,
ensuring type safety and preventing hardcoded string errors.
"""

import os
from enum import Enum
from typing import Optional


class WorkerType(str, Enum):
    """Worker types matching actual worker implementations.

    Values match Python module names (underscores, not hyphens).
    Directory names may differ (e.g., api-deployment vs api_deployment).
    """

    API_DEPLOYMENT = "api_deployment"
    GENERAL = "general"
    FILE_PROCESSING = "file_processing"
    CALLBACK = "callback"
    NOTIFICATION = "notification"
    LOG_CONSUMER = "log_consumer"
    SCHEDULER = "scheduler"
    EXECUTOR = "executor"
    IDE_CALLBACK = "ide_callback"

    @classmethod
    def from_directory_name(cls, name: str) -> "WorkerType":
        """Convert directory name to enum value.

        Args:
            name: Directory name (e.g., 'api-deployment', 'file_processing')

        Returns:
            Corresponding WorkerType enum

        Example:
            >>> WorkerType.from_directory_name("api-deployment")
            <WorkerType.API_DEPLOYMENT: 'api_deployment'>
        """
        normalized = name.replace("-", "_")
        return cls(normalized)

    def to_import_path(self) -> str:
        """Return standardized Python import path for tasks module.

        Returns:
            Import path string (e.g., 'api-deployment.tasks')

        Note:
            Handles the directory naming convention where some workers
            use hyphens in directory names but underscores in Python modules.
            Pluggable workers are loaded from pluggable_worker folder.
        """
        # Check if this is a pluggable worker
        if self.is_pluggable():
            return f"pluggable_worker.{self.value}.tasks"

        return f"{self.to_directory()}.tasks"

    def to_directory(self) -> str:
        """Return the on-disk directory name for this **non-pluggable** worker.

        Single source of truth for the naming-convention mapping: enum values use
        underscores (Python module names), while ``API_DEPLOYMENT``'s on-disk dir
        uses a hyphen (``api-deployment``). ``to_import_path`` builds on this, and
        the file-path task loader in ``worker.py`` reads it directly rather than
        slicing the import path.

        Raises:
            ValueError: if called on a pluggable worker type. Pluggable code lives
                under ``pluggable_worker/<value>``, so the mapping's passthrough
                would return a bare, *plausible-looking* but wrong path. Today only
                call ordering prevents that (``to_import_path`` guards with
                ``is_pluggable()``, and ``worker.py`` early-returns for pluggable
                workers before reaching here) — nothing enforced it. Since
                ``worker.py`` now RAISES on a missing directory instead of logging,
                a future caller that skips the guard would turn a silently-wrong
                path into a boot crash-loop; fail loudly and locally instead.
        """
        if self.is_pluggable():
            raise ValueError(
                f"{self!r} is a pluggable worker; its code lives under "
                f"pluggable_worker/{self.value} — use to_import_path() instead."
            )
        directory_mapping = {
            WorkerType.API_DEPLOYMENT: "api-deployment",
            # All others use same name for directory and module
        }
        return directory_mapping.get(self, self.value)

    def is_pluggable(self) -> bool:
        """Check if this worker type is a pluggable worker.

        Returns:
            bool: True if pluggable worker, False otherwise

        Note:
            OSS has no pluggable workers. Cloud imports this base
            and extends with pluggable worker logic.
        """
        # Base OSS version - no pluggable workers
        return False

    def to_worker_name(self) -> str:
        """Return human-readable worker name for logging.

        Returns:
            Worker name (e.g., 'api-deployment-worker')
        """
        return f"{self.value.replace('_', '-')}-worker"

    def to_health_port(self) -> int:
        """Return health check port for this worker.

        Checks environment variable first, then falls back to defaults.

        Returns:
            Port number for health server
        """
        # Check for environment variable first
        env_var = f"{self.name}_HEALTH_PORT"
        if env_port := os.getenv(env_var):
            try:
                return int(env_port)
            except ValueError:
                pass  # Fall back to default

        # Default port mapping
        port_mapping = {
            WorkerType.API_DEPLOYMENT: 8080,
            WorkerType.GENERAL: 8081,
            WorkerType.FILE_PROCESSING: 8082,
            WorkerType.CALLBACK: 8083,
            WorkerType.NOTIFICATION: 8085,
            WorkerType.LOG_CONSUMER: 8086,
            WorkerType.SCHEDULER: 8087,
            WorkerType.EXECUTOR: 8088,
            WorkerType.IDE_CALLBACK: 8089,
        }
        return port_mapping.get(self, 8080)


class QueueName(str, Enum):
    """Standard queue names used across the platform.

    These match the actual queue names in RabbitMQ/Redis.
    Using enums prevents typos and makes refactoring easier.
    """

    # Core queues
    API_DEPLOYMENTS = "celery_api_deployments"
    GENERAL = "celery"  # Default Celery queue

    # File processing queues
    FILE_PROCESSING = "file_processing"
    FILE_PROCESSING_API = "api_file_processing"

    # Callback queues
    CALLBACK = "file_processing_callback"
    CALLBACK_API = "api_file_processing_callback"

    # Notification queues
    NOTIFICATION = "notifications"
    NOTIFICATION_WEBHOOK = "notifications_webhook"
    NOTIFICATION_EMAIL = "notifications_email"
    NOTIFICATION_SMS = "notifications_sms"
    NOTIFICATION_PRIORITY = "notifications_priority"

    # Log processing queues
    LOG_CONSUMER = "celery_log_task_queue"
    PERIODIC_LOGS = "celery_periodic_logs"

    # Scheduler queue
    SCHEDULER = "scheduler"

    # Executor queue — queue-per-executor naming convention.
    # The dispatcher derives queue names as ``celery_executor_{executor_name}``.
    # The "legacy" executor is the default OSS executor.
    EXECUTOR = "celery_executor_legacy"
    EXECUTOR_AGENTIC = "celery_executor_agentic"

    # IDE callback queue (prompt studio post-execution callbacks)
    IDE_CALLBACK = "ide_callback"

    # Agent-KV terminal-callback queue (agent_kv_complete / agent_kv_error;
    # spec §5.3). Consumed by the IDE_CALLBACK worker alongside IDE_CALLBACK.
    AGENT_KV_CALLBACK = "agent_kv_callback"

    def to_env_var_name(self) -> str:
        """Convert queue name to environment variable name.

        Returns:
            Environment variable name (e.g., 'CELERY_API_DEPLOYMENTS_QUEUE')
        """
        return f"{self.value.upper().replace('-', '_')}_QUEUE"

    @classmethod
    def from_string(cls, queue_name: str) -> Optional["QueueName"]:
        """Safe conversion from string to QueueName enum.

        Args:
            queue_name: Queue name string

        Returns:
            QueueName enum or None if not found
        """
        for queue in cls:
            if queue.value == queue_name:
                return queue
        return None


class WorkerStatus(str, Enum):
    """Worker migration status for tracking."""

    NOT_MIGRATED = "not_migrated"
    IN_PROGRESS = "in_progress"
    MIGRATED = "migrated"
    VALIDATED = "validated"