LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
"""Worker Configuration Data Models
This module provides dataclasses for worker configuration, task routing,
and queue management. These replace dictionary-based configurations with
type-safe, validated dataclasses.
Migration Note: Part of the worker refactoring initiative to improve
code maintainability and reduce duplication across workers.
"""
import os
import sys
from collections.abc import Callable
from dataclasses import dataclass, field
from typing import Any
from shared.enums.worker_enums import QueueName, WorkerType
# Global cache for command-line Celery settings
_CMDLINE_CELERY_SETTINGS: dict[str, str] | None = None
def _parse_celery_cmdline_args() -> dict[str, str]:
"""Parse Celery settings from command-line arguments.
Extracts --setting=value pairs from sys.argv and converts them to
setting names compatible with get_celery_setting().
Returns:
Dictionary mapping setting names to values from command-line
Examples:
--pool=prefork -> {"POOL_TYPE": "prefork"}
--concurrency=20 -> {"CONCURRENCY": "20"}
--loglevel=INFO -> {"LOG_LEVEL": "INFO"}
"""
settings = {}
# Convert common Celery argument names to setting names
setting_mapping = {
"pool": "POOL_TYPE",
"concurrency": "CONCURRENCY",
"loglevel": "LOG_LEVEL",
"prefetch-multiplier": "PREFETCH_MULTIPLIER",
"max-tasks-per-child": "MAX_TASKS_PER_CHILD",
"time-limit": "TASK_TIME_LIMIT",
"soft-time-limit": "TASK_SOFT_TIME_LIMIT",
"without-gossip": "WORKER_GOSSIP",
"without-mingle": "WORKER_MINGLE",
"without-heartbeat": "WORKER_HEARTBEAT",
# Add more mappings as needed
}
for arg in sys.argv:
if arg.startswith("--"):
if "=" in arg:
# Handle --setting=value
setting_arg, value = arg[2:].split("=", 1)
setting_name = setting_mapping.get(
setting_arg, setting_arg.upper().replace("-", "_")
)
settings[setting_name] = value
elif arg[2:] in setting_mapping:
# Handle --without-* flags (they don't have values)
setting_arg = arg[2:]
if setting_arg.startswith("without-"):
setting_name = setting_mapping[setting_arg]
settings[setting_name] = "false"
return settings
def _get_cmdline_celery_settings() -> dict[str, str]:
"""Get cached command-line Celery settings, parsing if needed."""
global _CMDLINE_CELERY_SETTINGS
if _CMDLINE_CELERY_SETTINGS is None:
_CMDLINE_CELERY_SETTINGS = _parse_celery_cmdline_args()
return _CMDLINE_CELERY_SETTINGS
def get_celery_setting(
setting_name: str, worker_type: WorkerType, default: Any, setting_type: type = str
) -> Any:
"""Get Celery configuration setting with 4-tier hierarchy resolution.
Resolution order (most specific wins):
1. Command-line arguments (--pool=prefork, --concurrency=20, etc.) - HIGHEST
2. {WORKER_TYPE}_{SETTING_NAME} (worker-specific env override)
3. CELERY_{SETTING_NAME} (global env override)
4. default (Celery standard/provided default) - LOWEST
Args:
setting_name: The setting name (e.g., "TASK_TIME_LIMIT")
worker_type: Worker type for worker-specific overrides
default: Default value if no settings are found
setting_type: Type to convert the setting value to (str, int, bool, etc.)
Returns:
Resolved configuration value with proper type conversion
Examples:
# Check --pool=, then FILE_PROCESSING_POOL_TYPE, then CELERY_POOL_TYPE, then default
pool = get_celery_setting("POOL_TYPE", WorkerType.FILE_PROCESSING, "prefork", str)
# Check --concurrency=, then CALLBACK_CONCURRENCY, then CELERY_CONCURRENCY, then default
concurrency = get_celery_setting("CONCURRENCY", WorkerType.CALLBACK, 4, int)
"""
# 1. Check command-line arguments first (highest priority)
cmdline_settings = _get_cmdline_celery_settings()
if setting_name in cmdline_settings:
cmdline_value = cmdline_settings[setting_name]
converted_value = _convert_setting_value(cmdline_value, setting_type)
if converted_value is not None:
return converted_value
# 2. Check worker-specific setting (high priority)
worker_specific_key = f"{worker_type.name}_{setting_name}"
worker_value = os.getenv(worker_specific_key)
if worker_value is not None:
converted_value = _convert_setting_value(worker_value, setting_type)
if converted_value is not None:
return converted_value
# 3. Check global Celery setting (medium priority)
global_key = f"CELERY_{setting_name}"
global_value = os.getenv(global_key)
if global_value is not None:
converted_value = _convert_setting_value(global_value, setting_type)
if converted_value is not None:
return converted_value
# 4. Use provided default (lowest priority)
return default
def _convert_setting_value(value: str, setting_type: type) -> Any:
"""Convert string environment variable to appropriate type.
Args:
value: String value from environment variable
setting_type: Target type for conversion
Returns:
Converted value or None if empty/invalid
Raises:
ValueError: If conversion fails for non-empty values
"""
# Handle empty or whitespace-only values
if not value or not value.strip():
return None
# Clean the value
value = value.strip()
if setting_type == bool:
return value.lower() in ("true", "1", "yes", "on")
elif setting_type == int:
try:
return int(value)
except ValueError:
raise ValueError(f"Cannot convert '{value}' to integer")
elif setting_type == float:
try:
return float(value)
except ValueError:
raise ValueError(f"Cannot convert '{value}' to float")
elif setting_type == tuple:
# For autoscale tuples like "4,1" -> (4, 1)
try:
parts = [x.strip() for x in value.split(",") if x.strip()]
if not parts:
return None
return tuple(int(x) for x in parts)
except ValueError:
raise ValueError(f"Cannot convert '{value}' to tuple of integers")
else:
# Default to string (already cleaned)
return value
@dataclass
class TaskRoute:
"""Single task routing configuration.
Defines how a task pattern maps to a specific queue.
"""
pattern: str # Task name or pattern (e.g., "process_file_batch" or "worker.*")
queue: QueueName # Target queue enum
def to_dict(self) -> dict[str, str]:
"""Convert to Celery-compatible routing dict."""
return {"queue": self.queue.value}
@dataclass
class WorkerQueueConfig:
"""Queue configuration for a worker.
Defines primary and additional queues that a worker consumes from.
"""
primary_queue: QueueName
additional_queues: list[QueueName] = field(default_factory=list)
def all_queues(self) -> set[str]:
"""Get all queue names as strings.
Returns:
Set of queue name strings
"""
queues = {self.primary_queue.value}
queues.update(q.value for q in self.additional_queues)
return queues
def to_queue_list(self) -> list[str]:
"""Get ordered list of queue names.
Returns:
List with primary queue first, then additional queues
"""
result = [self.primary_queue.value]
result.extend(q.value for q in self.additional_queues)
return result
def to_cli_queues(self) -> str:
"""Format queues for Celery CLI --queues parameter.
Returns:
Comma-separated queue names
"""
return ",".join(self.to_queue_list())
@dataclass
class WorkerTaskRouting:
"""Complete task routing configuration for a worker.
Encapsulates all task routes for a specific worker type.
"""
worker_type: WorkerType
routes: list[TaskRoute]
def to_celery_config(self) -> dict[str, dict[str, str]]:
"""Convert to Celery task_routes configuration.
Returns:
Dictionary compatible with Celery's task_routes setting
"""
return {route.pattern: route.to_dict() for route in self.routes}
def add_route(self, pattern: str, queue: QueueName) -> None:
"""Add a new task route.
Args:
pattern: Task name or pattern
queue: Target queue
"""
self.routes.append(TaskRoute(pattern, queue))
def get_queue_for_task(self, task_name: str) -> QueueName | None:
"""Find queue for a specific task name.
Args:
task_name: Full task name
Returns:
Queue name if route found, None otherwise
"""
# Check exact matches first
for route in self.routes:
if route.pattern == task_name:
return route.queue
# Check wildcard patterns
for route in self.routes:
if route.pattern.endswith("*"):
prefix = route.pattern[:-1]
if task_name.startswith(prefix):
return route.queue
return None
@dataclass
class WorkerHealthConfig:
"""Health check configuration for a worker."""
port: int
custom_checks: list[Callable] = field(default_factory=list)
check_interval: int = 30 # seconds
def add_check(self, name: str, check_func: Callable) -> None:
"""Add a custom health check function."""
self.custom_checks.append((name, check_func))
@dataclass
class WorkerCeleryConfig:
"""Complete Celery configuration for a worker.
Combines all configuration aspects into a single dataclass.
"""
worker_type: WorkerType
queue_config: WorkerQueueConfig
task_routing: WorkerTaskRouting
health_config: WorkerHealthConfig | None = None
# Celery worker settings
prefetch_multiplier: int = 1
max_tasks_per_child: int = 1000
task_acks_late: bool = True
task_reject_on_worker_lost: bool = False
# Timeouts
task_time_limit: int = 7200 # 2 hours
task_soft_time_limit: int = 6300 # 1h 45m
# Retry settings
task_default_retry_delay: int = 60
task_max_retries: int = 3
# Pool settings
pool_type: str = "prefork" # or "threads", "solo", "eventlet"
concurrency: int | None = None # None means auto
autoscale: tuple[int, int] | None = None # (max, min) workers
# Task annotations (for task-specific settings like retry policies)
task_annotations: dict[str, dict[str, Any]] = field(default_factory=dict)
def _get_worker_specific_timeout_defaults(self) -> tuple[int, int]:
"""Get worker-specific timeout defaults.
Returns static defaults based on worker type that will be overridden
by environment variables like FILE_PROCESSING_TASK_TIME_LIMIT.
Returns:
tuple[int, int]: (hard_timeout, soft_timeout) in seconds
"""
# Worker-specific defaults (will be overridden by env vars)
if self.worker_type == WorkerType.FILE_PROCESSING:
return 7200, 6300 # 2 hours / 1h 45m (large files)
elif self.worker_type == WorkerType.CALLBACK:
return 3600, 3300 # 1 hour / 55 min
else:
# Conservative defaults for other workers
return 3600, 3300 # 1 hour / 55 min
def to_celery_dict(self, broker_url: str, result_backend: str) -> dict[str, Any]:
"""Generate complete Celery configuration dictionary with hierarchical resolution.
Uses 3-tier hierarchy: Worker-specific > Global > Default
Includes feature-specific logic for chord workers, pool types, etc.
Args:
broker_url: Celery broker URL
result_backend: Celery result backend URL
Returns:
Dictionary suitable for app.conf.update()
"""
config = {
# Connection settings
"broker_url": broker_url,
"result_backend": result_backend,
"broker_heartbeat": get_celery_setting(
"BROKER_HEARTBEAT", self.worker_type, 30, int
),
# Task routing
"task_routes": self.task_routing.to_celery_config(),
# Serialization (configurable from env)
"task_serializer": get_celery_setting(
"TASK_SERIALIZER", self.worker_type, "json"
),
"accept_content": ["json"], # Could make this configurable too
"result_serializer": get_celery_setting(
"RESULT_SERIALIZER", self.worker_type, "json"
),
"timezone": get_celery_setting("TIMEZONE", self.worker_type, "UTC"),
"enable_utc": get_celery_setting("ENABLE_UTC", self.worker_type, True, bool),
# Worker settings (all configurable from env with Celery defaults)
"worker_prefetch_multiplier": get_celery_setting(
"PREFETCH_MULTIPLIER",
self.worker_type,
1,
int, # Celery default is 4, but 1 is safer
),
"task_acks_late": get_celery_setting(
"TASK_ACKS_LATE", self.worker_type, True, bool
),
"worker_max_tasks_per_child": get_celery_setting(
"MAX_TASKS_PER_CHILD",
self.worker_type,
1000,
int, # Prevent memory leaks
),
"task_reject_on_worker_lost": get_celery_setting(
"TASK_REJECT_ON_WORKER_LOST",
self.worker_type,
self.task_reject_on_worker_lost,
bool,
),
"task_acks_on_failure_or_timeout": get_celery_setting(
"TASK_ACKS_ON_FAILURE_OR_TIMEOUT", self.worker_type, True, bool
),
"worker_disable_rate_limits": get_celery_setting(
"DISABLE_RATE_LIMITS", self.worker_type, False, bool
),
# Timeouts (configurable from env with worker-specific defaults)
# Falls back to task-specific timeout env vars (FILE_PROCESSING_TIMEOUT, etc.)
"task_time_limit": get_celery_setting(
"TASK_TIME_LIMIT",
self.worker_type,
self._get_worker_specific_timeout_defaults()[0],
int,
),
"task_soft_time_limit": get_celery_setting(
"TASK_SOFT_TIME_LIMIT",
self.worker_type,
self._get_worker_specific_timeout_defaults()[1],
int,
),
# Retry configuration (configurable from env)
"task_default_retry_delay": get_celery_setting(
"TASK_DEFAULT_RETRY_DELAY",
self.worker_type,
60,
int, # 1 minute default
),
"task_max_retries": get_celery_setting(
"TASK_MAX_RETRIES", self.worker_type, 3, int
),
# Stability (configurable from env)
"worker_pool_restarts": get_celery_setting(
"POOL_RESTARTS", self.worker_type, True, bool
),
"broker_connection_retry_on_startup": get_celery_setting(
"BROKER_CONNECTION_RETRY_ON_STARTUP", self.worker_type, True, bool
),
# Monitoring (configurable from env)
"worker_send_task_events": get_celery_setting(
"SEND_TASK_EVENTS", self.worker_type, True, bool
),
"task_send_sent_event": get_celery_setting(
"TASK_SEND_SENT_EVENT", self.worker_type, True, bool
),
# Task imports
"imports": [self.worker_type.to_import_path()],
}
# Feature-specific configurations
self._add_pool_configuration(config)
self._add_worker_scaling_configuration(config)
self._add_chord_configuration(config)
# Add task annotations if present
if self.task_annotations:
config["task_annotations"] = self.task_annotations
return config
def _add_pool_configuration(self, config: dict[str, Any]) -> None:
"""Add worker pool configuration based on worker type and environment."""
# All workers use prefork as default (consistent with command-line args)
default_pool = "prefork"
pool_type = get_celery_setting("POOL_TYPE", self.worker_type, default_pool)
if pool_type == "threads":
config["worker_pool"] = "threads"
# prefork is Celery's default, so no need to set it explicitly
def _add_worker_scaling_configuration(self, config: dict[str, Any]) -> None:
"""Add concurrency and autoscaling configuration."""
# Concurrency (fixed number of workers)
concurrency = get_celery_setting("CONCURRENCY", self.worker_type, None, int)
if concurrency is not None:
config["worker_concurrency"] = concurrency
# Autoscaling (dynamic worker count)
autoscale = get_celery_setting("AUTOSCALE", self.worker_type, None, tuple)
if autoscale is not None:
config["worker_autoscaler"] = "celery.worker.autoscale.Autoscaler"
config["worker_autoscale_max"] = autoscale[0]
config["worker_autoscale_min"] = autoscale[1]
def _add_chord_configuration(self, config: dict[str, Any]) -> None:
"""Add chord-specific configuration only for workers that use chords."""
# Only workers that actually use chords need chord configuration
chord_workers = {
WorkerType.CALLBACK,
WorkerType.GENERAL,
WorkerType.API_DEPLOYMENT,
}
if self.worker_type in chord_workers:
# Chord retry interval - defaults to Celery standard (1 second)
config["result_chord_retry_interval"] = get_celery_setting(
"RESULT_CHORD_RETRY_INTERVAL", self.worker_type, 1, float
)
def to_cli_args(self) -> list[str]:
"""Generate Celery worker CLI arguments.
Returns:
List of CLI arguments for celery worker command
"""
args = [
"worker",
"--loglevel=info",
f"--queues={self.queue_config.to_cli_queues()}",
]
if self.pool_type == "threads":
args.append("--pool=threads")
if self.concurrency:
args.append(f"--concurrency={self.concurrency}")
if self.autoscale:
args.append(f"--autoscale={self.autoscale[0]},{self.autoscale[1]}")
return args
@dataclass
class WorkerMigrationStatus:
"""Track migration status of a worker."""
worker_type: WorkerType
migrated: bool = False
validated: bool = False
notes: str = ""
def __post_init__(self):
"""Mark in worker file with __migrated__ flag."""
self.marker = f"__migrated__ = {self.migrated}"