LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Worker Type and Queue Name Enumerations
This module provides centralized enums for worker types and queue names,
ensuring type safety and preventing hardcoded string errors.
"""
import os
from enum import Enum
from typing import Optional
class WorkerType(str, Enum):
"""Worker types matching actual worker implementations.
Values match Python module names (underscores, not hyphens).
Directory names may differ (e.g., api-deployment vs api_deployment).
"""
API_DEPLOYMENT = "api_deployment"
GENERAL = "general"
FILE_PROCESSING = "file_processing"
CALLBACK = "callback"
NOTIFICATION = "notification"
LOG_CONSUMER = "log_consumer"
SCHEDULER = "scheduler"
EXECUTOR = "executor"
IDE_CALLBACK = "ide_callback"
@classmethod
def from_directory_name(cls, name: str) -> "WorkerType":
"""Convert directory name to enum value.
Args:
name: Directory name (e.g., 'api-deployment', 'file_processing')
Returns:
Corresponding WorkerType enum
Example:
>>> WorkerType.from_directory_name("api-deployment")
<WorkerType.API_DEPLOYMENT: 'api_deployment'>
"""
normalized = name.replace("-", "_")
return cls(normalized)
def to_import_path(self) -> str:
"""Return standardized Python import path for tasks module.
Returns:
Import path string (e.g., 'api-deployment.tasks')
Note:
Handles the directory naming convention where some workers
use hyphens in directory names but underscores in Python modules.
Pluggable workers are loaded from pluggable_worker folder.
"""
# Check if this is a pluggable worker
if self.is_pluggable():
return f"pluggable_worker.{self.value}.tasks"
return f"{self.to_directory()}.tasks"
def to_directory(self) -> str:
"""Return the on-disk directory name for this **non-pluggable** worker.
Single source of truth for the naming-convention mapping: enum values use
underscores (Python module names), while ``API_DEPLOYMENT``'s on-disk dir
uses a hyphen (``api-deployment``). ``to_import_path`` builds on this, and
the file-path task loader in ``worker.py`` reads it directly rather than
slicing the import path.
Raises:
ValueError: if called on a pluggable worker type. Pluggable code lives
under ``pluggable_worker/<value>``, so the mapping's passthrough
would return a bare, *plausible-looking* but wrong path. Today only
call ordering prevents that (``to_import_path`` guards with
``is_pluggable()``, and ``worker.py`` early-returns for pluggable
workers before reaching here) — nothing enforced it. Since
``worker.py`` now RAISES on a missing directory instead of logging,
a future caller that skips the guard would turn a silently-wrong
path into a boot crash-loop; fail loudly and locally instead.
"""
if self.is_pluggable():
raise ValueError(
f"{self!r} is a pluggable worker; its code lives under "
f"pluggable_worker/{self.value} — use to_import_path() instead."
)
directory_mapping = {
WorkerType.API_DEPLOYMENT: "api-deployment",
# All others use same name for directory and module
}
return directory_mapping.get(self, self.value)
def is_pluggable(self) -> bool:
"""Check if this worker type is a pluggable worker.
Returns:
bool: True if pluggable worker, False otherwise
Note:
OSS has no pluggable workers. Cloud imports this base
and extends with pluggable worker logic.
"""
# Base OSS version - no pluggable workers
return False
def to_worker_name(self) -> str:
"""Return human-readable worker name for logging.
Returns:
Worker name (e.g., 'api-deployment-worker')
"""
return f"{self.value.replace('_', '-')}-worker"
def to_health_port(self) -> int:
"""Return health check port for this worker.
Checks environment variable first, then falls back to defaults.
Returns:
Port number for health server
"""
# Check for environment variable first
env_var = f"{self.name}_HEALTH_PORT"
if env_port := os.getenv(env_var):
try:
return int(env_port)
except ValueError:
pass # Fall back to default
# Default port mapping
port_mapping = {
WorkerType.API_DEPLOYMENT: 8080,
WorkerType.GENERAL: 8081,
WorkerType.FILE_PROCESSING: 8082,
WorkerType.CALLBACK: 8083,
WorkerType.NOTIFICATION: 8085,
WorkerType.LOG_CONSUMER: 8086,
WorkerType.SCHEDULER: 8087,
WorkerType.EXECUTOR: 8088,
WorkerType.IDE_CALLBACK: 8089,
}
return port_mapping.get(self, 8080)
class QueueName(str, Enum):
"""Standard queue names used across the platform.
These match the actual queue names in RabbitMQ/Redis.
Using enums prevents typos and makes refactoring easier.
"""
# Core queues
API_DEPLOYMENTS = "celery_api_deployments"
GENERAL = "celery" # Default Celery queue
# File processing queues
FILE_PROCESSING = "file_processing"
FILE_PROCESSING_API = "api_file_processing"
# Callback queues
CALLBACK = "file_processing_callback"
CALLBACK_API = "api_file_processing_callback"
# Notification queues
NOTIFICATION = "notifications"
NOTIFICATION_WEBHOOK = "notifications_webhook"
NOTIFICATION_EMAIL = "notifications_email"
NOTIFICATION_SMS = "notifications_sms"
NOTIFICATION_PRIORITY = "notifications_priority"
# Log processing queues
LOG_CONSUMER = "celery_log_task_queue"
PERIODIC_LOGS = "celery_periodic_logs"
# Scheduler queue
SCHEDULER = "scheduler"
# Executor queue — queue-per-executor naming convention.
# The dispatcher derives queue names as ``celery_executor_{executor_name}``.
# The "legacy" executor is the default OSS executor.
EXECUTOR = "celery_executor_legacy"
EXECUTOR_AGENTIC = "celery_executor_agentic"
# IDE callback queue (prompt studio post-execution callbacks)
IDE_CALLBACK = "ide_callback"
# Agent-KV terminal-callback queue (agent_kv_complete / agent_kv_error;
# spec §5.3). Consumed by the IDE_CALLBACK worker alongside IDE_CALLBACK.
AGENT_KV_CALLBACK = "agent_kv_callback"
def to_env_var_name(self) -> str:
"""Convert queue name to environment variable name.
Returns:
Environment variable name (e.g., 'CELERY_API_DEPLOYMENTS_QUEUE')
"""
return f"{self.value.upper().replace('-', '_')}_QUEUE"
@classmethod
def from_string(cls, queue_name: str) -> Optional["QueueName"]:
"""Safe conversion from string to QueueName enum.
Args:
queue_name: Queue name string
Returns:
QueueName enum or None if not found
"""
for queue in cls:
if queue.value == queue_name:
return queue
return None
class WorkerStatus(str, Enum):
"""Worker migration status for tracking."""
NOT_MIGRATED = "not_migrated"
IN_PROGRESS = "in_progress"
MIGRATED = "migrated"
VALIDATED = "validated"