Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

52.4 KB
Raw Blame
#!/bin/bash
# =============================================================================
# Unstract Workers Runner Script
# =============================================================================
# This script provides a convenient way to run individual or multiple workers
# with proper environment configuration and health monitoring.

set -e

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color

# Script directory
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WORKERS_DIR="$SCRIPT_DIR"

# Default environment file
ENV_FILE="$WORKERS_DIR/.env"

# Worker type constant for the executor worker
readonly EXECUTOR_WORKER_TYPE="executor"
readonly IDE_CALLBACK_WORKER_TYPE="ide_callback"
# Canonical name of the PG-queue consumer worker (referenced in several maps
# and special-cases below; a constant keeps them in sync).
readonly PG_QUEUE_CONSUMER_TYPE="pg_queue_consumer"
# Canonical name of the PG-queue reaper (leader-elected recovery process).
readonly PG_QUEUE_REAPER_TYPE="pg_queue_reaper"
# Set alias that launches the whole PG-queue group (consumer + reaper) together.
readonly PG_QUEUE_SET="pg-queue"
# The PG-queue member worker dirs, as a set — the single source of truth for
# "which workers belong to the PG-queue transport". Keeps the `celery = all −
# pg-queue` complement correct by construction if a third member is ever added
# (callers test membership via ${PG_QUEUE_MEMBERS[$dir]:-} instead of
# hand-rolling the consumer/reaper pair).
# 9f: named PG-queue consumer ROLES. Each is a multi-queue consumer (one process
# draining several queues, mirroring the Celery file_processing/callback workers
# that each cover ETL+API) with its source worker-type + queue list baked in — so
# `./run-worker.sh -d pg-fileproc` needs no per-process env. Format:
# "<source_worker_type>;<comma,separated,queues>". The role name is passed as argv
# to `python -m pg_queue_consumer <role>` so pgrep (--status/-k/-r) tells the
# co-running roles apart (they are otherwise identical processes).
# The four coupled-pipeline stages, each a registry-bound consumer (mirroring the
# Celery api-deployment / general / file_processing / callback workers). The two
# orchestrator roles are split because async_execute_bin has DISTINCT impls per
# registry (api-deployment handles it as an API deployment; general routes by
# workflow type), so one consumer can't serve both — they bind different queues.
# fileproc/callback are multi-queue (one process drains both ETL + API queues).
# Role-name constants — the literal lives in one place and is referenced across
# PG_CONSUMER_ROLES / PG_QUEUE_MEMBERS / WORKERS.
readonly PG_ROLE_ORCH_API="pg-orchestrator-api"
readonly PG_ROLE_ORCH_GENERAL="pg-orchestrator-general"
readonly PG_ROLE_FILEPROC="pg-fileproc"
readonly PG_ROLE_CALLBACK="pg-callback"
readonly PG_ROLE_SCHEDULER="pg-scheduler"
readonly PG_ROLE_EXECUTOR="pg-executor"
readonly PG_ROLE_METRICS="pg-metrics"
readonly PG_ROLE_NOTIFICATION="pg-notification"
readonly PG_ROLE_IDE_CALLBACK="pg-ide-callback"
declare -rA PG_CONSUMER_ROLES=(
    ["$PG_ROLE_ORCH_API"]="api_deployment;celery_api_deployments"
    ["$PG_ROLE_ORCH_GENERAL"]="general;celery"
    ["$PG_ROLE_FILEPROC"]="file_processing;file_processing,file_processing_priority,api_file_processing"
    ["$PG_ROLE_CALLBACK"]="callback;file_processing_callback,celery_callback,api_file_processing_callback"
    # Runs scheduler.tasks.execute_pipeline_task fired by the orchestrator's PG
    # scheduler tick (Beat replacement). Distinct from the Celery 'scheduler'
    # worker (which Beat fires onto RabbitMQ).
    ["$PG_ROLE_SCHEDULER"]="scheduler;scheduler"
    # Runs execute_extraction (the executor RPC) over PG — request-reply: writes
    # the result to pg_task_result for the blocking caller. Queues mirror the
    # Celery executor's CELERY_QUEUES_EXECUTOR.
    ["$PG_ROLE_EXECUTOR"]="executor;celery_executor_legacy,celery_executor_agentic,celery_executor_table,celery_executor_smart_table,celery_executor_simple_prompt_studio,celery_executor_agentic_table,celery_executor_lookup_test"
    # Runs the dashboard_metrics.* periodics fired by the PG scheduler tick,
    # replacing the Celery 'workerMetrics' (-Q dashboard_metric_events). Its own
    # role rather than a queue bolted onto pg-scheduler: the consumer's health
    # heartbeat freezes for the duration of a task, so HEALTH_STALE is an upper
    # bound on one task's wall clock. The aggregation runs for minutes; sharing
    # pg-scheduler's tight 240s bound would restart that pod and take in-flight
    # pipeline triggers down with it. Sized in docker-compose.yaml.
    ["$PG_ROLE_METRICS"]="scheduler;dashboard_metric_events"
    # The backend notification seam enqueues send_webhook_notification onto
    # these queues unconditionally (UN-4046). Without this role a host-run PG
    # fleet accepts every buffered webhook and drains none — enqueue_task
    # succeeds, so nothing errors at the producer.
    ["$PG_ROLE_NOTIFICATION"]="notification;notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority"
    # Prompt Studio IDE callbacks (ide_index_*/ide_prompt_*/extraction_*).
    ["$PG_ROLE_IDE_CALLBACK"]="ide_callback;ide_callback,agent_kv_callback"
)
declare -rA PG_QUEUE_MEMBERS=(
    ["$PG_QUEUE_CONSUMER_TYPE"]=1
    ["$PG_QUEUE_REAPER_TYPE"]=1
    ["$PG_ROLE_ORCH_API"]=1
    ["$PG_ROLE_ORCH_GENERAL"]=1
    ["$PG_ROLE_FILEPROC"]=1
    ["$PG_ROLE_CALLBACK"]=1
    ["$PG_ROLE_SCHEDULER"]=1
    ["$PG_ROLE_EXECUTOR"]=1
    ["$PG_ROLE_METRICS"]=1
    ["$PG_ROLE_NOTIFICATION"]=1
    ["$PG_ROLE_IDE_CALLBACK"]=1
)
# The Celery transport set: every worker EXCEPT the PG-queue members — the
# *complement* of the 'pg-queue' set, so the two transports' logs can be tailed
# separately (-L celery vs -L pg-queue). Since 9f, 'celery' is BOTH a -L tail
# alias AND a run alias (WORKERS maps it to "all" → run_all_workers), symmetric
# with 'pg'; 'all' remains its synonym. Either way it omits the opt-in PG workers.
readonly CELERY_SET="celery"

# Available workers
declare -A WORKERS=(
    ["api"]="api-deployment"
    ["api-deployment"]="api-deployment"
    ["general"]="general"
    ["file"]="file_processing"
    ["file-processing"]="file_processing"
    ["callback"]="callback"
    ["log"]="log_consumer"
    ["log-consumer"]="log_consumer"
    ["logs"]="log_consumer"
    ["notification"]="notification"
    ["notifications"]="notification"
    ["notify"]="notification"
    ["scheduler"]="scheduler"
    ["schedule"]="scheduler"
    ["${EXECUTOR_WORKER_TYPE}"]="${EXECUTOR_WORKER_TYPE}"
    ["ide-callback"]="${IDE_CALLBACK_WORKER_TYPE}"
    ["${IDE_CALLBACK_WORKER_TYPE}"]="${IDE_CALLBACK_WORKER_TYPE}"
    # PG Queue consumer — polls Postgres (SKIP LOCKED), not RabbitMQ via Celery
    ["pg-queue-consumer"]="$PG_QUEUE_CONSUMER_TYPE"
    ["$PG_QUEUE_CONSUMER_TYPE"]="$PG_QUEUE_CONSUMER_TYPE"
    ["pg-consumer"]="$PG_QUEUE_CONSUMER_TYPE"
    # PG Queue named roles (9f) — coupled-pipeline consumers, queues + source
    # worker-type baked in. Run individually like any worker, or via the 'pg' set.
    ["$PG_ROLE_ORCH_API"]="$PG_ROLE_ORCH_API"
    ["$PG_ROLE_ORCH_GENERAL"]="$PG_ROLE_ORCH_GENERAL"
    ["$PG_ROLE_FILEPROC"]="$PG_ROLE_FILEPROC"
    ["$PG_ROLE_CALLBACK"]="$PG_ROLE_CALLBACK"
    ["$PG_ROLE_SCHEDULER"]="$PG_ROLE_SCHEDULER"
    ["$PG_ROLE_EXECUTOR"]="$PG_ROLE_EXECUTOR"
    ["$PG_ROLE_METRICS"]="$PG_ROLE_METRICS"
    # PG Queue reaper — leader-elected recovery loop (barrier-orphan sweep)
    ["reaper"]="$PG_QUEUE_REAPER_TYPE"
    ["pg-queue-reaper"]="$PG_QUEUE_REAPER_TYPE"
    ["$PG_QUEUE_REAPER_TYPE"]="$PG_QUEUE_REAPER_TYPE"
    # Set: launch the whole PG-queue group (consumer + reaper) in one shot
    ["$PG_QUEUE_SET"]="$PG_QUEUE_SET"
    ["pg"]="$PG_QUEUE_SET"
    ["all"]="all"
    # 'celery' is a run alias for the Celery set (== 'all', which excludes the PG
    # workers) — symmetric with 'pg'. Maps to "all" so it dispatches to
    # run_all_workers and list_core_worker_dirs skips it (not a phantom dir).
    ["$CELERY_SET"]="all"
)

# Pluggable workers will be auto-discovered at runtime
declare -A PLUGGABLE_WORKERS=()

# Worker queue mappings
declare -A WORKER_QUEUES=(
    ["api-deployment"]="celery_api_deployments"
    ["general"]="celery"
    ["file_processing"]="file_processing,api_file_processing"
    ["callback"]="file_processing_callback,api_file_processing_callback"
    ["log_consumer"]="celery_log_task_queue"
    ["notification"]="notifications,notifications_webhook,notifications_email,notifications_sms,notifications_priority"
    ["scheduler"]="scheduler"
    ["${EXECUTOR_WORKER_TYPE}"]="celery_executor_legacy"
    # agent_kv_callback carries the Agent-KV terminal callbacks
    # (agent_kv_complete/agent_kv_error, spec §5.3) dispatched by
    # backend/agent_kv/dispatch.py; ide_callback owns both queues.
    ["${IDE_CALLBACK_WORKER_TYPE}"]="${IDE_CALLBACK_WORKER_TYPE},agent_kv_callback"
    # The PG queue (in pg_queue_message) this consumer polls — exported as
    # WORKER_PG_QUEUE_CONSUMER_QUEUE, not a Celery --queues value.
    ["$PG_QUEUE_CONSUMER_TYPE"]="notifications"
)

# Worker health ports
declare -A WORKER_HEALTH_PORTS=(
    ["api-deployment"]="8080"
    ["general"]="8081"
    ["file_processing"]="8082"
    ["callback"]="8083"
    ["log_consumer"]="8084"
    ["notification"]="8085"
    ["scheduler"]="8087"
    ["${EXECUTOR_WORKER_TYPE}"]="8088"
    ["${IDE_CALLBACK_WORKER_TYPE}"]="8089"
    # pg_queue_consumer: 8090 — reserved here, just past the 8080-8089 core
    # range and just below where pluggable-worker discovery starts allocating
    # (8091+, see below), so it collides with neither. The consumer binds it
    # only when WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT is exported (below); a bare
    # `python -m pg_queue_consumer` binds nothing.
    ["$PG_QUEUE_CONSUMER_TYPE"]="8090"
    # pg_queue_reaper: 8086 — the one free slot in the 8080-8090 band (8086 sits
    # between callback's 8085 and scheduler's 8087). Bound only when
    # WORKER_PG_REAPER_HEALTH_PORT is exported (below); a bare
    # `python -m pg_queue_reaper` binds nothing.
    ["$PG_QUEUE_REAPER_TYPE"]="8086"
)

# Opt-in workers: experimental and NOT part of the default "all" fleet, so
# they're started only on explicit request. Status shows them only when they
# are actually running, so a deliberate non-start isn't reported as a STOPPED
# failure (they'd otherwise show STOPPED after every `all`).
declare -A OPTIN_WORKERS=(
    ["$PG_QUEUE_CONSUMER_TYPE"]=1
    ["$PG_QUEUE_REAPER_TYPE"]=1
)

# Function to display usage
usage() {
    cat << EOF
Usage: $0 [OPTIONS] WORKER_TYPE

Run Unstract Celery workers with proper environment configuration.

WORKER_TYPE:
    api, api-deployment    Run API deployment worker
    general               Run general worker (webhooks, background tasks)
    file, file-processing Run file processing worker
    callback              Run callback worker
    log, log-consumer     Run log consumer worker
    notification, notify  Run notification worker
    scheduler, schedule   Run scheduler worker (scheduled pipeline tasks)
    executor              Run executor worker (extraction execution tasks)
    ide-callback          Run IDE callback worker (Prompt Studio post-execution callbacks)
    pg-queue-consumer     Run a generic PG-queue poll-loop consumer (env-configured; opt-in)
    pg-orchestrator-api   Run the PG orchestrator consumer for API execs (celery_api_deployments)
    pg-orchestrator-general Run the PG orchestrator consumer for ETL/general execs (celery)
    pg-fileproc           Run the PG fan-out consumer (file_processing + api_file_processing)
    pg-callback           Run the PG callback consumer (file_processing_callback + api_file_processing_callback)
    pg-scheduler          Run the PG scheduler consumer (runs execute_pipeline_task fired by the orchestrator)
    reaper, pg-queue-reaper Run PG-queue reaper (leader-elected recovery; opt-in)
    pg, pg-queue          Run the whole PG-queue set (the 4 pipeline roles + reaper) together
    all, celery           Run the Celery worker set (all Celery workers; excludes the PG set)

Note: Pluggable workers in pluggable_worker/ directory are automatically discovered and can be run by name.
Note: 'all' (alias 'celery') is the Celery worker set; 'pg' ('pg-queue') is the PG-queue set. They are
      independent — pass both (e.g. `-d all pg`) for a dual-transport (strangler-fig) setup.
Note: pg-queue-consumer overrides: WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE (source worker whose
      tasks to load, default notification), WORKER_PG_QUEUE_CONSUMER_QUEUE (queue to poll),
      and WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT (liveness server port, default 8090).
Note: reaper overrides: WORKER_PG_ORCHESTRATOR_LEASE_SECONDS (lease window, default 10),
      WORKER_PG_REAPER_INTERVAL_SECONDS (cycle interval, default 5),
      WORKER_PG_REAPER_HEALTH_PORT (liveness server port, default 8086),
      WORKER_PG_REAPER_HEALTH_STALE_SECONDS (liveness staleness window, default 30).

OPTIONS:
    -e, --env-file FILE   Use specific environment file (default: .env)
    -d, --detach          Run worker in background (daemon mode)
    -l, --log-level LEVEL Set log level (DEBUG, INFO, WARNING, ERROR)
    -c, --concurrency N   Set worker concurrency (default: auto)
    -q, --queues QUEUES   Override default queues (comma-separated)
    -p, --health-port N   Override health check port
    -P, --pool TYPE       Set Celery pool type (threads, prefork, gevent, solo, eventlet)
    -n, --hostname NAME   Set custom worker hostname/name
    -k, --kill            Kill running workers and exit
    -r, --restart         Kill matching worker(s) then relaunch (with WORKER_TYPE,
                          restarts only that worker; without it, restarts all)
    -s, --status          Show status of running workers
    -L, --logs [WORKER]   Live-tail worker log files (all if WORKER omitted).
                          WORKER may also be a set: 'celery' (all but the
                          PG-queue members) or 'pg-queue' (consumer + reaper).
    -C, --clear-logs      Delete worker .log files created by -d / 'all' runs
    -h, --help            Show this help message

EXAMPLES:
    # Run API deployment worker
    $0 api

    # Run general worker with debug logging
    $0 -l DEBUG general

    # Run file processing worker in background
    $0 -d file

    # Start BOTH transports in one shot (Celery set + PG set) — use -d so neither blocks
    $0 -d all pg

    # Run with custom environment file
    $0 -e production.env all

    # Run with custom concurrency
    $0 -c 4 general

    # Run with custom worker name (useful for scaling)
    $0 -n api-01 api
    $0 -n api-02 api

    # Run with specific pool type
    $0 -P prefork file
    $0 --pool threads general

    # Check worker status
    $0 -s

    # Kill all running workers
    $0 -k

    # Live-tail all worker logs
    $0 -L

    # Tail just the Celery set's logs (excludes the PG-queue workers)
    $0 -L celery

    # Tail just the PG-queue set's logs (consumer + reaper)
    $0 -L pg-queue

    # Tail just one worker's log
    $0 -L general

    # Wipe out old log files
    $0 -C

    # Restart all workers
    $0 -r -P prefork

    # Restart just one worker
    $0 -r general -l DEBUG

ENVIRONMENT:
    The script will load environment variables from .env file if present.
    Required variables:
    - INTERNAL_SERVICE_API_KEY
    - INTERNAL_API_BASE_URL
    - CELERY_BROKER_BASE_URL (not needed when LOG_TRANSPORT=redis and only PG-queue
      workers are started)
    - DB_HOST, DB_USER, DB_PASSWORD, DB_NAME (for PostgreSQL result backend)

    Plugin availability is detected dynamically via plugin registry.
    See sample.env for full configuration options.

HEALTH CHECKS:
    Workers can optionally bind an HTTP health server on the port assigned
    via {WORKER}_HEALTH_PORT env vars (8080-8089 by default). It's used by
    K8s liveness probes in production but is NOT load-bearing locally —
    'run-worker.sh -s' only reports whether the Celery process is alive.

EOF
}

# Function to print colored output
print_status() {
    local color=$1
    local message=$2
    echo -e "${color}${message}${NC}"
}

# Function to load environment file
load_env() {
    local env_file=$1

    if [[ -f "$env_file" ]]; then
        print_status $GREEN "Loading environment from: $env_file"
        set -a  # automatically export all variables
        source "$env_file"
        set +a
    else
        print_status $YELLOW "Warning: Environment file not found: $env_file"
        print_status $YELLOW "Make sure required environment variables are set"
    fi
}

# Function to discover pluggable workers
discover_pluggable_workers() {
    local pluggable_dir="$WORKERS_DIR/pluggable_worker"

    # Silently return if pluggable_worker directory doesn't exist
    if [[ ! -d "$pluggable_dir" ]]; then
        return
    fi

    local discovered_count=0

    # Scan for directories with worker.py
    for worker_path in "$pluggable_dir"/*; do
        # Skip if not a directory
        if [[ ! -d "$worker_path" ]]; then
            continue
        fi

        local worker_name=$(basename "$worker_path")

        # Skip special directories (starting with _ or .)
        if [[ "$worker_name" == _* ]] || [[ "$worker_name" == .* ]]; then
            continue
        fi

        # Check if worker.py exists
        if [[ -f "$worker_path/worker.py" ]]; then
            # Register the pluggable worker
            PLUGGABLE_WORKERS["$worker_name"]="$worker_name"

            # Add hyphenated alias for convenience (e.g., bulk-download -> bulk_download)
            local hyphenated_name="${worker_name//_/-}"
            if [[ "$hyphenated_name" != "$worker_name" ]]; then
                PLUGGABLE_WORKERS["$hyphenated_name"]="$worker_name"
            fi

            # Add shorthand alias (e.g., bulk -> bulk_download)
            local first_part="${worker_name%%_*}"
            if [[ "$first_part" != "$worker_name" ]]; then
                PLUGGABLE_WORKERS["$first_part"]="$worker_name"
            fi

            # Set default queue if not already defined
            if [[ -z "${WORKER_QUEUES[$worker_name]:-}" ]]; then
                WORKER_QUEUES["$worker_name"]="$worker_name"
            fi

            # Assign health port dynamically (starting from 8091; 8090 is
            # reserved for pg_queue_consumer, so the first pluggable worker
            # doesn't collide with it).
            if [[ -z "${WORKER_HEALTH_PORTS[$worker_name]:-}" ]]; then
                WORKER_HEALTH_PORTS["$worker_name"]=$((8091 + discovered_count))
            fi

            print_status $GREEN "Discovered pluggable worker: $worker_name"
            ((discovered_count++)) || true
        fi
    done

    if [[ $discovered_count -gt 0 ]]; then
        print_status $BLUE "Total pluggable workers: $discovered_count"
    fi
}

# Function to validate environment
validate_env() {
    local required_vars=(
        "INTERNAL_SERVICE_API_KEY"
        "INTERNAL_API_BASE_URL"
    )

    # The broker is needed by every Celery worker, and by every worker while logs
    # are published over Celery (LOG_TRANSPORT other than "redis", the default).
    # Skip it only when logs go over Redis and every requested worker is a
    # PG-queue member (or the PG-queue set itself).
    local log_transport="${LOG_TRANSPORT-celery}"
    log_transport="${log_transport,,}"
    log_transport="${log_transport#"${log_transport%%[![:space:]]*}"}"
    log_transport="${log_transport%"${log_transport##*[![:space:]]}"}"
    local needs_broker=""
    if [[ "$log_transport" != "redis" ]]; then
        needs_broker=1
    fi
    local wt resolved
    for wt in "${WORKER_TYPES[@]}"; do
        resolved="${WORKERS[$wt]:-${PLUGGABLE_WORKERS[$wt]:-}}"
        if [[ "$resolved" != "$PG_QUEUE_SET" && -z "${PG_QUEUE_MEMBERS[$resolved]:-}" ]]; then
            needs_broker=1
            break
        fi
    done
    if [[ -n "$needs_broker" ]]; then
        required_vars+=("CELERY_BROKER_BASE_URL")
    fi

    required_vars+=(
        "DB_HOST"
        "DB_USER"
        "DB_PASSWORD"
        "DB_NAME"
    )

    local missing_vars=()

    for var in "${required_vars[@]}"; do
        if [[ -z "${!var}" ]]; then
            missing_vars+=("$var")
        fi
    done

    if [[ ${#missing_vars[@]} -gt 0 ]]; then
        print_status $RED "Error: Missing required environment variables:"
        for var in "${missing_vars[@]}"; do
            print_status $RED "  - $var"
        done
        print_status $YELLOW "Please check your .env file or set these variables manually"
        exit 1
    fi
}

# Function to get worker PIDs
# Anchors the match on the hostname value's start (preceded by a non-word
# character) so e.g. "callback" doesn't also match "ide_callback". The
# trailing `(@|-)` covers both Celery hostname forms emitted by run_worker:
#   --hostname=callback-worker@%h         (default, no WORKER_INSTANCE_ID)
#   --hostname=callback-worker-${id}@%h   (when WORKER_INSTANCE_ID is set)
get_worker_pids() {
    local worker_type=$1 pattern out rc
    # The PG-queue consumer and reaper run as `python -m <pkg>`, not a Celery
    # `<type>-worker@host` process, so they have no `-worker` token to anchor on.
    # Match the module invocation instead (covers both the `uv run python` parent
    # and the `python -m` child). Keeps --status / -k / -r working for them.
    if [[ -n "${PG_CONSUMER_ROLES[$worker_type]:-}" ]]; then
        # 9f roles run as `python -m pg_queue_consumer <role>` — anchor on the
        # role argv so co-running roles (and the generic consumer) don't collide.
        pattern="-m[[:space:]]+${PG_QUEUE_CONSUMER_TYPE}[[:space:]]+${worker_type}([[:space:]]|\$)"
    elif [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then
        # End-anchored so the GENERIC consumer doesn't also match the role
        # consumers (which run as `... -m pg_queue_consumer <role>`); they have
        # their own role-anchored pattern above. The reaper takes no role arg, so
        # the same end anchor matches it cleanly.
        pattern="-m[[:space:]]+${worker_type}[[:space:]]*\$"
    else
        pattern="[^[:alnum:]_]${worker_type}-worker(@|-)"
    fi
    # pgrep exits 1 for "no match" (normal — absorbed) but >=2 for an
    # operational/regex error. Distinguish them: collapsing rc>=2 to empty would
    # make a live worker look absent (a -k no-op, or a duplicate spawn on -r).
    out=$(pgrep -f -- "$pattern")
    rc=$?
    if (( rc > 1 )); then
        print_status "$YELLOW" "warning: pgrep failed (rc=$rc) while matching $worker_type" >&2
    fi
    [[ -n "$out" ]] && printf '%s\n' "$out"
    return 0
}

# Returns get_worker_pids output as a single space-separated string with
# the trailing space stripped — the form xargs and `kill` want. Wrapping
# this avoids repeating the `tr | sed` pipeline at every caller.
get_worker_pids_oneline() {
    local worker_type=$1
    get_worker_pids "$worker_type" | tr '\n' ' ' | sed 's/ $//'
}

# Function to resolve canonical worker dir names from the WORKERS map
# (skips aliases and "all"). Includes discovered pluggable workers.
list_core_worker_dirs() {
    local seen=""
    for key in "${!WORKERS[@]}"; do
        local value="${WORKERS[$key]}"
        # Skip set aliases ("all", "pg-queue") — they're groups, not real worker
        # dirs/processes, so status must not list them as phantom STOPPED workers.
        if [[ "$value" == "all" || "$value" == "$PG_QUEUE_SET" ]]; then
            continue
        fi
        if [[ "$seen" == *" $value "* ]]; then
            continue
        fi
        seen="$seen $value "
        echo "$value"
    done
}

list_pluggable_worker_dirs() {
    for key in "${!PLUGGABLE_WORKERS[@]}"; do
        local value="${PLUGGABLE_WORKERS[$key]}"
        if [[ "$key" == "$value" ]]; then
            echo "$value"
        fi
    done
}

# Function to resolve the log file path for a given canonical worker dir.
# Core workers live at WORKERS_DIR/$dir/$dir.log; pluggable workers
# live at WORKERS_DIR/pluggable_worker/$dir/$dir.log.
resolve_log_file() {
    local worker_dir=$1
    local core_path="$WORKERS_DIR/$worker_dir/$worker_dir.log"
    local pluggable_path="$WORKERS_DIR/pluggable_worker/$worker_dir/$worker_dir.log"
    # 9f roles run from (and log under) the pg_queue_consumer launcher dir as
    # "<role>.log", since they share that dir rather than having their own.
    local role_path="$WORKERS_DIR/$PG_QUEUE_CONSUMER_TYPE/$worker_dir.log"
    if [[ -f "$core_path" ]]; then
        echo "$core_path"
    elif [[ -f "$pluggable_path" ]]; then
        echo "$pluggable_path"
    elif [[ -n "${PG_CONSUMER_ROLES[$worker_dir]:-}" && -f "$role_path" ]]; then
        echo "$role_path"
    fi
}

# Function to tail one or all worker log files (-L|--logs)
tail_logs() {
    local requested=$1  # may be empty → tail all
    local log_files=()

    if [[ -z "$requested" || "$requested" == "all" || "$requested" == "$CELERY_SET" ]]; then
        # 'all' (and empty) tails every worker; 'celery' tails every worker
        # EXCEPT the PG-queue members — the complement of the 'pg-queue' alias
        # (handled in the else-branch) — so the two transports' logs can be
        # tailed separately. Same resolve-and-append body for both, single-sourced.
        for d in $(list_core_worker_dirs) $(list_pluggable_worker_dirs); do
            if [[ "$requested" == "$CELERY_SET" && -n "${PG_QUEUE_MEMBERS[$d]:-}" ]]; then
                continue
            fi
            local f
            f=$(resolve_log_file "$d")
            [[ -n "$f" ]] && log_files+=("$f")
        done
    else
        # Resolve alias (e.g. "api" → "api-deployment") via WORKERS / PLUGGABLE_WORKERS
        local canonical="${WORKERS[$requested]:-${PLUGGABLE_WORKERS[$requested]:-}}"
        if [[ -z "$canonical" || "$canonical" == "all" ]]; then
            print_status $RED "Error: Unknown worker type for logs: $requested"
            print_status $BLUE "Tip: omit the worker type to tail all logs"
            exit 1
        fi
        if [[ "$canonical" == "$PG_QUEUE_SET" ]]; then
            # The set alias maps to no single dir — tail every member's log
            # (the named role consumers + the reaper).
            for d in "${!PG_CONSUMER_ROLES[@]}" "$PG_QUEUE_REAPER_TYPE"; do
                local member_log
                member_log=$(resolve_log_file "$d")
                [[ -n "$member_log" ]] && log_files+=("$member_log")
            done
        else
            local f
            f=$(resolve_log_file "$canonical")
            if [[ -z "$f" ]]; then
                print_status $YELLOW "No log file found for $canonical. Did you start it with -d?"
                exit 0
            fi
            log_files+=("$f")
        fi
    fi

    if [[ ${#log_files[@]} -eq 0 ]]; then
        print_status $YELLOW "No log files found. Workers must be started in detached mode (-d or 'all') for logs to be written to files."
        exit 0
    fi

    print_status $BLUE "Tailing ${#log_files[@]} log file(s) — Ctrl+C to stop"
    for f in "${log_files[@]}"; do
        print_status $GREEN "  $f"
    done
    exec tail -F "${log_files[@]}"
}

# Function to delete worker log files (-C|--clear-logs)
clear_logs() {
    local count=0
    for d in $(list_core_worker_dirs) $(list_pluggable_worker_dirs); do
        local f
        f=$(resolve_log_file "$d")
        if [[ -n "$f" ]]; then
            rm -f "$f"
            print_status $GREEN "Removed: $f"
            ((count++)) || true
        fi
    done
    if [[ $count -eq 0 ]]; then
        print_status $YELLOW "No worker log files to clear"
    else
        print_status $BLUE "Cleared $count log file(s)"
    fi
}

# Function to kill a single worker by its canonical dir name.
# Uses the anchored matcher from get_worker_pids so callback vs ide_callback
# don't bleed into each other. Surfaces kill failures (EPERM, ESRCH) and
# verifies the processes are gone before reporting success.
kill_one_worker() {
    local worker_dir=$1
    local pids
    pids=$(get_worker_pids_oneline "$worker_dir")
    if [[ -z "$pids" ]]; then
        print_status $YELLOW "  $worker_dir: not running"
        return 0
    fi
    print_status $YELLOW "  $worker_dir: killing $pids"
    # shellcheck disable=SC2086
    kill -TERM $pids || print_status $RED "  $worker_dir: kill -TERM failed"
    sleep 1
    local survivors
    survivors=$(get_worker_pids_oneline "$worker_dir")
    if [[ -n "$survivors" ]]; then
        # shellcheck disable=SC2086
        kill -KILL $survivors || print_status $RED "  $worker_dir: kill -KILL failed"
        sleep 1
        survivors=$(get_worker_pids_oneline "$worker_dir")
        if [[ -n "$survivors" ]]; then
            print_status $RED "  $worker_dir: survivors after SIGKILL: $survivors"
            return 1
        fi
    fi
    return 0
}

# Function to kill all known workers. Iterates the canonical dir list and
# delegates to kill_one_worker so the anchored matcher is the single source
# of truth — avoids killing unrelated `celery worker` processes on the host.
kill_workers() {
    print_status $YELLOW "Killing all running workers..."
    local any_failed=0
    local workers_to_kill
    workers_to_kill="$(list_core_worker_dirs) $(list_pluggable_worker_dirs)"
    for worker in $workers_to_kill; do
        kill_one_worker "$worker" || any_failed=1
    done
    if [[ $any_failed -eq 0 ]]; then
        print_status $GREEN "All workers stopped"
    else
        print_status $RED "One or more workers could not be stopped"
        return 1
    fi
}

# Function to show worker status
show_status() {
    print_status $BLUE "Worker Status:"
    echo "=============="

    # Derive list from WORKERS (skips aliases & "all") so adding a new
    # worker in one place keeps status in sync. Pluggable workers included.
    local workers_to_check
    workers_to_check="$(list_core_worker_dirs) $(list_pluggable_worker_dirs)"

    for worker in $workers_to_check; do
        local pids
        pids=$(get_worker_pids_oneline "$worker")

        # Opt-in workers aren't part of `all`; only surface them when running
        # so an intentional non-start doesn't read as a STOPPED failure.
        if [[ -z "$pids" && -n "${OPTIN_WORKERS[$worker]:-}" ]]; then
            continue
        fi

        printf '  %-22s ' "$worker:"

        if [[ -n "$pids" ]]; then
            local count
            count=$(echo "$pids" | wc -w)
            print_status $GREEN "RUNNING ($count proc, PID: $pids)"
        else
            print_status $RED "STOPPED"
        fi
    done
}

# Function to run a single worker
run_worker() {
    local worker_type=$1
    local detach=$2
    local log_level=$3
    local concurrency=$4
    local custom_queues=$5
    local health_port=$6
    local pool_type=$7
    local custom_hostname=$8

    # Determine worker directory (handle pluggable workers)
    local worker_dir
    if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" ]]; then
        # Pluggable worker - use subdirectory
        worker_dir="$WORKERS_DIR/pluggable_worker/$worker_type"
    elif [[ -n "${PG_CONSUMER_ROLES[$worker_type]:-}" ]]; then
        # 9f role consumers share the pg_queue_consumer launcher dir (there is no
        # per-role directory; the role only selects queues + source worker-type).
        worker_dir="$WORKERS_DIR/$PG_QUEUE_CONSUMER_TYPE"
    else
        # Core worker - use root directory
        worker_dir="$WORKERS_DIR/$worker_type"
    fi

    if [[ ! -d "$worker_dir" ]]; then
        print_status $RED "Error: Worker directory not found: $worker_dir"

        # Provide helpful message for pluggable workers
        if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" ]]; then
            echo ""
            echo -e "${YELLOW}This is a cloud-only pluggable worker.${NC}"
            echo -e "Make sure:"
            echo -e "  1. The pluggable_worker/$worker_type folder exists"
            echo -e "  2. The worker module has a valid worker.py file"
            echo ""
        fi

        exit 1
    fi

    # Set worker-specific environment variables
    export WORKER_NAME="${worker_type}-worker"
    export WORKER_TYPE="$(echo "$worker_type" | tr '-' '_')"  # Convert hyphens to underscores for Python module names
    export LOG_LEVEL="${log_level:-INFO}"

    # Set health port if specified
    if [[ -n "$health_port" ]]; then
        case "$worker_type" in
            "api-deployment")
                export API_DEPLOYMENT_HEALTH_PORT="$health_port"
                ;;
            "general")
                export GENERAL_HEALTH_PORT="$health_port"
                ;;
            "file_processing")
                export FILE_PROCESSING_HEALTH_PORT="$health_port"
                ;;
            "callback")
                export CALLBACK_HEALTH_PORT="$health_port"
                ;;
            "log_consumer")
                export LOG_CONSUMER_HEALTH_PORT="$health_port"
                ;;
            "notification")
                export NOTIFICATION_HEALTH_PORT="$health_port"
                ;;
            "scheduler")
                export SCHEDULER_HEALTH_PORT="$health_port"
                ;;
            "${EXECUTOR_WORKER_TYPE}")
                export EXECUTOR_HEALTH_PORT="$health_port"
                ;;
            "${IDE_CALLBACK_WORKER_TYPE}")
                export IDE_CALLBACK_HEALTH_PORT="$health_port"
                ;;
            *)
                # Handle pluggable workers dynamically
                if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" ]]; then
                    worker_type_upper=$(echo "$worker_type" | tr '[:lower:]' '[:upper:]' | tr '-' '_')
                    export "${worker_type_upper}_HEALTH_PORT=$health_port"
                fi
                ;;
        esac
    fi

    # Determine queues
    local queues="${custom_queues:-${WORKER_QUEUES[$worker_type]}}"

    # Build meaningful worker name
    local worker_instance_name="${worker_type}-worker"
    if [[ -n "$custom_hostname" ]]; then
        worker_instance_name="$custom_hostname"
    elif [[ -n "$WORKER_INSTANCE_ID" ]]; then
        worker_instance_name="${worker_type}-worker-${WORKER_INSTANCE_ID}"
    fi

    # Determine celery app module path (handle pluggable workers)
    local celery_app_module
    if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" ]]; then
        # Pluggable worker - use full module path
        celery_app_module="pluggable_worker.${worker_type}.worker"
    else
        # Core worker - use worker module
        celery_app_module="worker"
    fi

    # Build celery command
    local cmd_args=(
        "uv" "run" "celery" "-A" "$celery_app_module" "worker"
        "--loglevel=${log_level:-info}"
        "--queues=$queues"
        "--hostname=${worker_instance_name}@%h"
    )

    # Add pool type if specified
    if [[ -n "$pool_type" ]]; then
        cmd_args+=("--pool=$pool_type")
    fi

    # Add concurrency if specified
    if [[ -n "$concurrency" ]]; then
        cmd_args+=("--concurrency=$concurrency")
    fi

    # Add concurrency for production-like setup
    if [[ -z "$concurrency" ]]; then
        case "$worker_type" in
            "api-deployment")
                cmd_args+=("--concurrency=2")
                ;;
            "general")
                cmd_args+=("--concurrency=4")
                ;;
            "file_processing")
                cmd_args+=("--concurrency=4")
                ;;
            "callback")
                cmd_args+=("--concurrency=4")
                ;;
            "log_consumer")
                cmd_args+=("--concurrency=2")
                ;;
            "notification")
                cmd_args+=("--concurrency=2")
                ;;
            "scheduler")
                cmd_args+=("--concurrency=2")
                ;;
            "${EXECUTOR_WORKER_TYPE}")
                cmd_args+=("--concurrency=2")
                ;;
            "${IDE_CALLBACK_WORKER_TYPE}")
                cmd_args+=("--concurrency=2")
                ;;
            *)
                # Default for pluggable and other workers
                if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" ]]; then
                    cmd_args+=("--concurrency=2")
                fi
                ;;
        esac
    fi

    # PG queue consumer is a plain Python poll-loop (polls Postgres via
    # SKIP LOCKED, not a Celery/RabbitMQ worker) — override the celery command
    # with the bootstrapping launcher and route the queue via env.
    pg_consumer_role="${PG_CONSUMER_ROLES[$worker_type]:-}"
    if [[ "$worker_type" == "$PG_QUEUE_CONSUMER_TYPE" || -n "$pg_consumer_role" ]]; then
        # A named 9f role bakes in its source worker-type + multi-queue list
        # ("<type>;<queues>"); the generic pg-queue-consumer reads them from env
        # (default source worker = notification, the first migrated leaf task).
        if [[ -n "$pg_consumer_role" ]]; then
            # Plain assignment — the line below is the single export point (its
            # :- default/override is load-bearing for the generic consumer path).
            WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE="${pg_consumer_role%%;*}"
            queues="${pg_consumer_role#*;}"
        fi
        export WORKER_PG_QUEUE_CONSUMER_QUEUE="$queues"
        export WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE="${WORKER_PG_QUEUE_CONSUMER_WORKER_TYPE:-notification}"
        # Liveness HTTP server port (-p override wins, else the map default; roles
        # have no map entry → empty → the launcher skips the health server).
        export WORKER_PG_QUEUE_CONSUMER_HEALTH_PORT="${health_port:-${WORKER_HEALTH_PORTS[$worker_type]:-}}"
        cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_CONSUMER_TYPE")
        # Tag the process with the role name so pgrep (--status/-k/-r) can tell
        # co-running roles apart — they are otherwise identical `python -m` procs.
        [[ -n "$pg_consumer_role" ]] && cmd_args+=("$worker_type")
    fi

    # PG queue reaper — a leader-elected SQL recovery loop (no Celery, no task
    # bootstrap). Override the celery command with the plain `python -m` entry.
    # Tunables (lease window, cycle interval) come from env. The liveness server
    # binds when WORKER_PG_REAPER_HEALTH_PORT is set (-p override wins, else the
    # map default) — exported so the reaper's main() opts in.
    if [[ "$worker_type" == "$PG_QUEUE_REAPER_TYPE" ]]; then
        export WORKER_PG_REAPER_HEALTH_PORT="${health_port:-${WORKER_HEALTH_PORTS[$worker_type]}}"
        cmd_args=("uv" "run" "python" "-m" "$PG_QUEUE_REAPER_TYPE")
    fi

    print_status $GREEN "Starting $worker_type worker..."
    print_status $BLUE "Directory: $worker_dir"
    print_status $BLUE "Worker Name: $worker_instance_name"
    print_status $BLUE "Queues: ${queues:-n/a}"
    # Show the effective port: a -p/--health-port override wins over the map.
    print_status $BLUE "Health Port: ${health_port:-${WORKER_HEALTH_PORTS[$worker_type]:-n/a}}"
    print_status $BLUE "Command: ${cmd_args[*]}"

    # Change to appropriate directory
    # For pluggable workers, stay at workers root to allow module imports
    # For core workers, change to worker directory
    if [[ -n "${PLUGGABLE_WORKERS[$worker_type]:-}" || -n "${PG_QUEUE_MEMBERS[$worker_type]:-}" ]]; then
        # Run from the workers root so `python -m pg_queue_consumer` /
        # `python -m pg_queue_reaper` (and what they import) resolve. PG_QUEUE_MEMBERS
        # is the single source of truth for "which workers are PG-queue members".
        cd "$WORKERS_DIR"
    else
        cd "$worker_dir"
    fi

    if [[ "$detach" == "true" ]]; then
        # Run in background. Write to an ABSOLUTE log path ($worker_dir is
        # absolute) so the file lands where resolve_log_file() / -L / -C look,
        # regardless of cwd. Workers that run from the workers root (pluggable
        # workers, pg_queue_consumer) would otherwise drop a relative
        # "$worker_type.log" at the root, where -L/-C can't find it.
        local log_file="$worker_dir/$worker_type.log"
        nohup "${cmd_args[@]}" > "$log_file" 2>&1 &
        local pid=$!
        # set -e does not apply to backgrounded jobs, so a fork that dies on
        # startup (e.g. the consumer's require_tasks RuntimeError, an
        # `import worker` failure, a bad env cast) would still be reported as
        # "started" — and for pg_queue_consumer, which has no health port, a
        # dead process then just reads as absent in --status. Catch an
        # *immediate* exit. This is a best-effort fast-fail for crash-on-import
        # / bad-config faults (sub-second), NOT a connectivity check: a worker
        # that dies slowly (e.g. a broker connect timing out after >1s) still
        # passes here and surfaces later via its health port / --status. Run all
        # detached workers share it — an immediate crash can hit any worker type;
        # in `all` the subshells are backgrounded, so this 1s overlaps the
        # inter-launch sleep rather than serializing.
        sleep 1
        if ! kill -0 "$pid" 2>/dev/null; then
            print_status $RED "$worker_type worker failed to start (PID $pid exited) — last log lines:"
            tail -n 20 "$log_file" 2>/dev/null
            return 1
        fi
        print_status $GREEN "$worker_type worker started in background (PID: $pid)"
        print_status $BLUE "Logs: $log_file"
    else
        # Run in foreground
        exec "${cmd_args[@]}"
    fi
}

# Function to run all workers
run_all_workers() {
    local detach=$1
    local log_level=$2
    local concurrency=$3
    local pool_type=$4

    print_status $GREEN "Starting all workers..."

    # Define core workers
    local core_workers="api-deployment general file_processing callback log_consumer notification scheduler executor ide_callback"

    # Add discovered pluggable workers
    if [[ ${#PLUGGABLE_WORKERS[@]} -gt 0 ]]; then
        print_status $BLUE "Including pluggable workers in 'all' mode"

        # Get unique pluggable worker names (skip aliases)
        local pluggable_list=""
        for key in "${!PLUGGABLE_WORKERS[@]}"; do
            local value="${PLUGGABLE_WORKERS[$key]}"
            # Only add if it's the canonical name (not an alias)
            if [[ "$key" == "$value" ]]; then
                pluggable_list="$pluggable_list $value"
            fi
        done

        core_workers="$core_workers$pluggable_list"
    fi

    # Always run all workers in background when using "all"
    for worker in $core_workers; do
        print_status $BLUE "Starting $worker worker in background..."

        # Run each worker in background
        (
            run_worker "$worker" "true" "$log_level" "$concurrency" "" "" "$pool_type" ""
        ) &

        sleep 2  # Give each worker time to start
    done

    if [[ "$detach" != "true" ]]; then
        print_status $GREEN "All workers started. Press Ctrl+C to stop all workers."
        print_status $BLUE "Worker status:"
        sleep 3
        show_status

        # Wait for any background job to finish (they won't unless killed)
        wait
    else
        print_status $GREEN "All workers started in background"
        show_status
    fi
}

# Function to run the PG-queue worker set (the 4 pipeline roles + reaper) together.
# A set is multiple processes, so — like 'all' — it ALWAYS runs detached
# (it ignores -d/--detach; there's no foreground form). The PG-queue counterpart
# to 'all' (the Celery set): run both for a dual-transport (strangler-fig) setup.
# The reaper is a leader-elected singleton, so running it on several hosts is
# safe (only one wins the lease). Returns non-zero if any member dies on start —
# the reaper has no health port yet, so this launch check is the only
# programmatic startup signal a caller (systemd/CI) gets.
run_pg_queue_set() {
    local log_level=$1
    local concurrency=$2
    local pool_type=$3

    # The set = the named pipeline consumer roles (9f) + the reaper. The fan-out
    # and callback roles are multi-queue (ETL+API); the two orchestrator roles
    # each bind a single registry-specific queue (see PG_CONSUMER_ROLES).
    local members=("${!PG_CONSUMER_ROLES[@]}" "$PG_QUEUE_REAPER_TYPE")
    print_status $GREEN "Starting PG-queue set (${members[*]})..."
    local failed=0
    for worker in "${members[@]}"; do
        print_status $BLUE "Starting $worker in background..."
        # A FOREGROUND subshell: run_worker (detach=true) nohup-backgrounds the
        # actual worker and returns 1 on an immediate crash-on-start — so the
        # subshell's exit status IS that signal. The subshell isolates run_worker's
        # `cd` from this loop; the nohup'd worker survives the subshell exiting.
        # (A background `( … ) &` would lose the status — its `$!` is the launcher
        # subshell, which exits the instant it backgrounds the worker.)
        ( run_worker "$worker" "true" "$log_level" "$concurrency" "" "" "$pool_type" "" ) \
            || failed=1
    done
    if [[ $failed -ne 0 ]]; then
        print_status $RED "PG-queue set: a member failed to start — tearing down the set (see logs above)"
        # Don't leave a survivor: a restart-on-failure relaunch would spawn a
        # second instance on top of it (a consumer would double-poll Postgres).
        # Kill all members (a crashed one is already gone → no-op). Same
        # all-or-nothing discipline as the restart path — aggregate kill failures
        # so a survivor (would double-poll Postgres) is surfaced, not silent.
        local teardown_failed=0
        for worker in "${members[@]}"; do
            kill_one_worker "$worker" || teardown_failed=1
        done
        if [[ $teardown_failed -ne 0 ]]; then
            print_status $RED "PG-queue set: a member survived SIGKILL during teardown — check for a duplicate consumer double-polling Postgres"
        fi
        show_status
        return 1
    fi
    print_status $GREEN "PG-queue set started in background"
    show_status
}

# Parse command line arguments
DETACH=false
LOG_LEVEL=""
CONCURRENCY=""
CUSTOM_QUEUES=""
HEALTH_PORT=""
POOL_TYPE=""
CUSTOM_HOSTNAME=""
KILL_WORKERS=false
SHOW_STATUS=false
LOGS_MODE=false
CLEAR_LOGS_MODE=false
RESTART_MODE=false
# Multiple positional worker types may be given (e.g. `all pg` to start both the
# Celery and PG sets in one shot). WORKER_TYPE stays the first for the
# single-target paths (-r/validation); the launch path loops WORKER_TYPES.
WORKER_TYPES=()

while [[ $# -gt 0 ]]; do
    case $1 in
        -e|--env-file)
            ENV_FILE="$2"
            shift 2
            ;;
        -d|--detach)
            DETACH=true
            shift
            ;;
        -L|--logs)
            LOGS_MODE=true
            shift
            ;;
        -C|--clear-logs)
            CLEAR_LOGS_MODE=true
            shift
            ;;
        -l|--log-level)
            LOG_LEVEL="$2"
            shift 2
            ;;
        -c|--concurrency)
            CONCURRENCY="$2"
            shift 2
            ;;
        -q|--queues)
            CUSTOM_QUEUES="$2"
            shift 2
            ;;
        -p|--health-port)
            HEALTH_PORT="$2"
            shift 2
            ;;
        -P|--pool)
            POOL_TYPE="$2"
            shift 2
            ;;
        -n|--hostname)
            CUSTOM_HOSTNAME="$2"
            shift 2
            ;;
        -k|--kill)
            KILL_WORKERS=true
            shift
            ;;
        -r|--restart)
            RESTART_MODE=true
            shift
            ;;
        -s|--status)
            SHOW_STATUS=true
            shift
            ;;
        -h|--help)
            usage
            exit 0
            ;;
        -*)
            print_status $RED "Unknown option: $1"
            usage
            exit 1
            ;;
        *)
            WORKER_TYPES+=("$1")
            shift
            ;;
    esac
done

# First positional drives the single-target paths (-r, validation messages); the
# launch path below iterates all of WORKER_TYPES.
WORKER_TYPE="${WORKER_TYPES[0]:-}"

# Handle special actions
if [[ "$KILL_WORKERS" == "true" ]]; then
    kill_workers
    exit 0
fi

if [[ "$SHOW_STATUS" == "true" ]]; then
    discover_pluggable_workers
    show_status
    exit 0
fi

if [[ "$LOGS_MODE" == "true" ]]; then
    discover_pluggable_workers
    tail_logs "$WORKER_TYPE"
    exit 0
fi

if [[ "$CLEAR_LOGS_MODE" == "true" ]]; then
    discover_pluggable_workers
    clear_logs
    exit 0
fi

# Restart = kill matching + fall through to the normal launch path.
# - If WORKER_TYPE is set (and not "all"), kill only that worker.
# - Otherwise kill everything; the launch path will treat the unset
#   WORKER_TYPE as "all" once we set it below.
if [[ "$RESTART_MODE" == "true" ]]; then
    discover_pluggable_workers
    if [[ "${WORKERS[$WORKER_TYPE]:-}" == "$PG_QUEUE_SET" ]]; then
        # Restart the PG-queue set: kill both members, then fall through to the
        # launch path (which runs run_pg_queue_set since WORKER_TYPE is the set).
        # Aggregate kill failures (kill_one_worker returns 1 if a process survives
        # SIGKILL) and abort rather than relaunch over a survivor — a second
        # consumer would double-poll Postgres. Mirrors kill_workers' discipline.
        print_status $BLUE "Restarting PG-queue set..."
        restart_failed=0
        # Kill the SAME members run_pg_queue_set launches (the named roles +
        # reaper) — not the generic consumer — or a survivor + relaunch would
        # double-poll Postgres.
        for pg_member in "${!PG_CONSUMER_ROLES[@]}" "$PG_QUEUE_REAPER_TYPE"; do
            kill_one_worker "$pg_member" || restart_failed=1
        done
        if [[ $restart_failed -ne 0 ]]; then
            print_status $RED "Cannot restart PG-queue set: a member survived SIGKILL; aborting to avoid duplicate processes"
            exit 1
        fi
    elif [[ -n "$WORKER_TYPE" && "$WORKER_TYPE" != "all" && "${WORKERS[$WORKER_TYPE]:-}" != "all" ]]; then
        # ('celery' maps to "all" → falls through to the restart-all branch below.)
        restart_target_dir="${WORKERS[$WORKER_TYPE]:-${PLUGGABLE_WORKERS[$WORKER_TYPE]:-}}"
        if [[ -z "$restart_target_dir" ]]; then
            print_status $RED "Error: Unknown worker type for restart: $WORKER_TYPE"
            exit 1
        fi
        print_status $BLUE "Restarting $restart_target_dir..."
        kill_one_worker "$restart_target_dir"
    else
        kill_workers
        WORKER_TYPE="all"
    fi
fi

# A worker type is required. (restart-all set WORKER_TYPE=all without a positional
# → seed WORKER_TYPES from it so the launch loop below has a target.)
if [[ ${#WORKER_TYPES[@]} -eq 0 ]]; then
    if [[ -n "$WORKER_TYPE" ]]; then
        WORKER_TYPES=("$WORKER_TYPE")
    else
        print_status $RED "Error: Worker type is required"
        usage
        exit 1
    fi
fi

# Load environment
load_env "$ENV_FILE"

# Discover pluggable workers
discover_pluggable_workers

# Validate every requested worker type (core or pluggable) up front.
for wt in "${WORKER_TYPES[@]}"; do
    if [[ -z "${WORKERS[$wt]:-}" && -z "${PLUGGABLE_WORKERS[$wt]:-}" ]]; then
        print_status $RED "Error: Unknown worker type: $wt"
        print_status $BLUE "Available core workers: ${!WORKERS[*]}"
        if [[ ${#PLUGGABLE_WORKERS[@]} -gt 0 ]]; then
            # Show unique pluggable worker names (not aliases)
            pluggable_names=""
            for key in "${!PLUGGABLE_WORKERS[@]}"; do
                value="${PLUGGABLE_WORKERS[$key]}"
                if [[ "$key" == "$value" ]]; then
                    pluggable_names="$pluggable_names $value"
                fi
            done
            print_status $BLUE "Available pluggable workers:$pluggable_names"
        fi
        exit 1
    fi
done

# Validate environment
validate_env

# Add PYTHONPATH for imports
export PYTHONPATH="$WORKERS_DIR:${PYTHONPATH:-}"

# Multi-type launches (e.g. `all pg`) must be detached: a non-detached set
# (run_all_workers) `wait`s on its background jobs and would block any later type.
if [[ ${#WORKER_TYPES[@]} -gt 1 && "$DETACH" != "true" ]]; then
    print_status $YELLOW "Multiple worker types given without -d/--detach: a foreground set will block the rest. Re-run with -d to start them all."
fi

# Run each requested worker / set in turn.
for wt in "${WORKER_TYPES[@]}"; do
    if [[ "${WORKERS[$wt]:-}" == "all" ]]; then
        # 'all' and its synonym 'celery' (the Celery set, excludes PG workers).
        run_all_workers "$DETACH" "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE"
    elif [[ "${WORKERS[$wt]:-}" == "$PG_QUEUE_SET" ]]; then
        # The PG-queue set (pipeline roles + reaper). Always backgrounded (multiple
        # procs). Propagate a member start-failure to the script exit code — the
        # reaper has no health port, so this is the only programmatic startup signal.
        run_pg_queue_set "$LOG_LEVEL" "$CONCURRENCY" "$POOL_TYPE" || exit 1
    else
        # Resolve worker directory name from either WORKERS or PLUGGABLE_WORKERS
        worker_dir_name="${WORKERS[$wt]:-${PLUGGABLE_WORKERS[$wt]}}"
        run_worker "$worker_dir_name" "$DETACH" "$LOG_LEVEL" "$CONCURRENCY" "$CUSTOM_QUEUES" "$HEALTH_PORT" "$POOL_TYPE" "$CUSTOM_HOSTNAME"
    fi
done