Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

2.6 KB
Raw Blame
"""Abort active workflow runs during Celery warm shutdown."""

import logging
import threading
from typing import Any

from celery.signals import worker_shutdown, worker_shutting_down

from core.app.apps.workflow.active_workflow_tasks import (
    get_active_workflow_task_count,
    reset_active_workflow_tasks,
)

logger = logging.getLogger(__name__)
WORKFLOW_WARM_SHUTDOWN_ABORT_REASON = "Workflow stopped because the worker is shutting down."
_WORKER_SHUTTING_DOWN_DISPATCH_UID = "dify.workflow_warm_shutdown.shutting_down"
_WORKER_SHUTDOWN_DISPATCH_UID = "dify.workflow_warm_shutdown.shutdown"
_celery_warm_shutdown_started = threading.Event()


def _is_warm_shutdown(how: Any) -> bool:
    return str(how).strip().lower() == "warm"


def celery_warm_shutdown_started() -> bool:
    """Return whether the current worker process started Celery warm shutdown."""
    return _celery_warm_shutdown_started.is_set()


def mark_celery_warm_shutdown_started() -> None:
    """Mark the current worker process as being in Celery warm shutdown."""
    _celery_warm_shutdown_started.set()


def _on_worker_shutting_down(*args: object, **kwargs: object) -> None:
    """Mark warm shutdown and log the active workflow run count."""
    how = kwargs.get("how")
    if not _is_warm_shutdown(how):
        logger.debug("Skip workflow abort during non-warm Celery shutdown: how=%s", how)
        return

    mark_celery_warm_shutdown_started()
    abort_count = get_active_workflow_task_count()
    if abort_count == 0:
        logger.info("No active workflow runs found during Celery warm shutdown")
        return

    logger.info(
        "Marked Celery warm shutdown for %s active workflow run(s)",
        abort_count,
    )


def _on_worker_shutdown(*args: object, **kwargs: object) -> None:
    """Log whether tracked workflow tasks ended before Celery worker shutdown."""
    remaining_run_count = get_active_workflow_task_count()
    if remaining_run_count:
        logger.warning(
            "Celery worker is shutting down with %s workflow run(s) still active after warm shutdown wait",
            remaining_run_count,
        )
        return

    logger.info("Celery worker shutdown reached after all tracked workflow runs ended")


def setup_workflow_warm_shutdown_handler() -> None:
    """Connect Celery worker shutdown handlers for workflow abort and logging."""
    reset_active_workflow_tasks()
    worker_shutting_down.connect(
        _on_worker_shutting_down,
        weak=False,
        dispatch_uid=_WORKER_SHUTTING_DOWN_DISPATCH_UID,
    )
    worker_shutdown.connect(
        _on_worker_shutdown,
        weak=False,
        dispatch_uid=_WORKER_SHUTDOWN_DISPATCH_UID,
    )