Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

2.4 KB
Raw Blame
"""Executor Worker

Celery worker for the pluggable executor system.
Routes execute_extraction tasks to registered executors.
"""

import os

from queue_backend import worker_task
from shared.enums.worker_enums import WorkerType
from shared.infrastructure.config.builder import WorkerBuilder
from shared.infrastructure.config.registry import WorkerRegistry
from shared.infrastructure.logging import WorkerLogger

# Setup worker
logger = WorkerLogger.setup(WorkerType.EXECUTOR)
app, config = WorkerBuilder.build_celery_app(WorkerType.EXECUTOR)


def check_executor_health():
    """Custom health check for executor worker."""
    from shared.infrastructure.monitoring.health import (
        HealthCheckResult,
        HealthStatus,
    )

    try:
        from unstract.sdk1.execution.registry import (
            ExecutorRegistry,
        )

        executors = ExecutorRegistry.list_executors()

        return HealthCheckResult(
            name="executor_health",
            status=HealthStatus.HEALTHY,
            message="Executor worker is healthy",
            details={
                "worker_type": "executor",
                "registered_executors": executors,
                "executor_count": len(executors),
                "queues": os.environ.get(
                    "CELERY_QUEUES_EXECUTOR", "celery_executor_legacy"
                ).split(","),
            },
        )

    except Exception as e:
        return HealthCheckResult(
            name="executor_health",
            status=HealthStatus.DEGRADED,
            message=f"Health check failed: {e}",
            details={"error": str(e)},
        )


# Register health check
WorkerRegistry.register_health_check(
    WorkerType.EXECUTOR,
    "executor_health",
    check_executor_health,
)


@worker_task(bind=True)
def healthcheck(self):
    """Health check task for monitoring systems."""
    return {
        "status": "healthy",
        "worker_type": "executor",
        "task_id": self.request.id,
        "worker_name": (config.worker_name if config else "executor-worker"),
    }


# Import tasks so shared_task definitions bind to this app. This also populates
# ExecutorRegistry: executor/tasks.py calls register_all() at module scope, and
# it is the site that does the work on every deployed path — workers/worker.py
# exec-loads that file by path, and nothing launches `celery -A executor`.
import executor.tasks  # noqa: E402, F401