Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): stop leaking Redis result connections from fire-and-forget groups (#44551)

Yunlu Wen903ca7478ec2
diff --git a/api/schedule/trigger_provider_refresh_task.py b/api/schedule/trigger_provider_refresh_task.py
index b7b9d67da27..4845162127f 100644
--- a/api/schedule/trigger_provider_refresh_task.py
+++ b/api/schedule/trigger_provider_refresh_task.py
@@ -2,7 +2,6 @@ import logging
import math
from collections.abc import Iterable, Sequence
-from celery import group
from sqlalchemy import ColumnElement, and_, func, or_, select
from sqlalchemy.engine.row import Row
from sqlalchemy.orm import Session
@@ -90,17 +89,18 @@ def trigger_provider_refresh() -> None:
for (tenant_id, subscription_id), is_locked in zip(subscriptions, acquired, strict=False)
if is_locked
]
- result = group(jobs).apply_async()
+ with app.celery.producer_or_acquire() as producer: # type: ignore
+ for job in jobs:
+ job.apply_async(producer=producer)
enqueued = len(jobs)
logger.info(
- "Trigger refresh page %d/%d: scanned=%d locks_acquired=%d enqueued=%d result=%s",
+ "Trigger refresh page %d/%d: scanned=%d locks_acquired=%d enqueued=%d",
page + 1,
pages,
len(subscriptions),
sum(1 for x in acquired if x),
enqueued,
- result,
)
logger.info("Trigger refresh scan done: due=%d", total_due)
diff --git a/api/schedule/workflow_schedule_task.py b/api/schedule/workflow_schedule_task.py
index 2fee9e467d5..4427f6d4e02 100644
--- a/api/schedule/workflow_schedule_task.py
+++ b/api/schedule/workflow_schedule_task.py
@@ -1,6 +1,6 @@
import logging
-from celery import current_app, group, shared_task
+from celery import current_app, shared_task
from sqlalchemy import and_, select
from sqlalchemy.orm import Session, sessionmaker
@@ -102,8 +102,8 @@ def _process_schedules(session: Session, schedules: list[WorkflowSchedulePlan],
tasks_to_dispatch.append(schedule.id)
if tasks_to_dispatch:
- job = group(run_schedule_trigger.s(schedule_id) for schedule_id in tasks_to_dispatch)
- job.apply_async(producer=producer)
+ for schedule_id in tasks_to_dispatch:
+ run_schedule_trigger.apply_async(args=(schedule_id,), producer=producer)
logger.debug("Dispatched %d tasks in parallel", len(tasks_to_dispatch))
diff --git a/api/tasks/rag_pipeline/rag_pipeline_run_task.py b/api/tasks/rag_pipeline/rag_pipeline_run_task.py
index 251c439e9ca..26f2a0247d8 100644
--- a/api/tasks/rag_pipeline/rag_pipeline_run_task.py
+++ b/api/tasks/rag_pipeline/rag_pipeline_run_task.py
@@ -8,7 +8,7 @@ from itertools import islice
from typing import Any
import click
-from celery import group, shared_task
+from celery import shared_task
from flask import current_app, g
from pydantic import TypeAdapter
from sqlalchemy import select
@@ -114,8 +114,8 @@ def rag_pipeline_run_task(
)
)
- if jobs:
- group(jobs).apply_async()
+ for job in jobs:
+ job.apply_async()
else:
# No more waiting tasks, clear the flag
tenant_isolated_task_queue.delete_task_key()
diff --git a/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py b/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py
index 545391e4e7a..91792e6f81d 100644
--- a/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py
+++ b/api/tests/test_containers_integration_tests/tasks/test_rag_pipeline_run_tasks.py
@@ -1,8 +1,9 @@
import json
import uuid
-from unittest.mock import MagicMock, patch
+from unittest.mock import patch
import pytest
+from celery.canvas import Signature
from faker import Faker
from flask import Flask
from sqlalchemy.orm import Session
@@ -389,10 +390,8 @@ class TestRagPipelineRunTasks:
# Set the task key to indicate there are waiting tasks (legacy behavior)
redis_client.set(legacy_task_key, 1, ex=60 * 60)
- # Mock the Celery group scheduling used by the implementation
- with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
- mock_group.return_value.apply_async = MagicMock()
-
+ # Mock the Celery task publishing used by the implementation
+ with patch.object(Signature, "apply_async", autospec=True) as mock_apply_async:
# Act: Execute the priority task with new code but legacy queue data
rag_pipeline_run_task(file_id, tenant.id)
@@ -401,11 +400,11 @@ class TestRagPipelineRunTasks:
mock_file_service["delete_file"].assert_called_once_with(file_id)
assert mock_pipeline_generator.call_count == 1
- # Verify waiting tasks were processed via group, pull 1 task a time by default
- assert mock_group.return_value.apply_async.called
+ # Verify waiting tasks were processed via apply_async, pull 1 task a time by default
+ assert mock_apply_async.called
# Verify correct parameters for the first scheduled job signature
- jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+ jobs = [call.args[0] for call in mock_apply_async.call_args_list]
first_kwargs = jobs[0].kwargs if jobs else {}
assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == legacy_file_ids[0]
assert first_kwargs.get("tenant_id") == tenant.id
@@ -450,10 +449,8 @@ class TestRagPipelineRunTasks:
waiting_file_ids = [str(uuid.uuid4()) for _ in range(3)]
queue.push_tasks(waiting_file_ids)
- # Mock the Celery group scheduling used by the implementation
- with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
- mock_group.return_value.apply_async = MagicMock()
-
+ # Mock the Celery task publishing used by the implementation
+ with patch.object(Signature, "apply_async", autospec=True) as mock_apply_async:
# Act: Execute the regular task
rag_pipeline_run_task(file_id, tenant.id)
@@ -462,11 +459,11 @@ class TestRagPipelineRunTasks:
mock_file_service["delete_file"].assert_called_once_with(file_id)
assert mock_pipeline_generator.call_count == 1
- # Verify waiting tasks were processed via group.apply_async
- assert mock_group.return_value.apply_async.called
+ # Verify waiting tasks were processed via apply_async
+ assert mock_apply_async.called
# Verify correct parameters for the first scheduled job signature
- jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+ jobs = [call.args[0] for call in mock_apply_async.call_args_list]
first_kwargs = jobs[0].kwargs if jobs else {}
assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_ids[0]
assert first_kwargs.get("tenant_id") == tenant.id
@@ -564,10 +561,8 @@ class TestRagPipelineRunTasks:
waiting_file_id = str(uuid.uuid4())
queue.push_tasks([waiting_file_id])
- # Mock the Celery group scheduling used by the implementation
- with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
- mock_group.return_value.apply_async = MagicMock()
-
+ # Mock the Celery task publishing used by the implementation
+ with patch.object(Signature, "apply_async", autospec=True) as mock_apply_async:
# Act: Execute the regular task (should not raise exception)
rag_pipeline_run_task(file_id, tenant.id)
@@ -578,10 +573,10 @@ class TestRagPipelineRunTasks:
assert mock_pipeline_generator.call_count == 1
# Verify waiting task was still processed despite core processing error
- assert mock_group.return_value.apply_async.called
+ assert mock_apply_async.called
# Verify correct parameters for the first scheduled job signature
- jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+ jobs = [call.args[0] for call in mock_apply_async.call_args_list]
first_kwargs = jobs[0].kwargs if jobs else {}
assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id
assert first_kwargs.get("tenant_id") == tenant.id
@@ -694,10 +689,8 @@ class TestRagPipelineRunTasks:
queue1.push_tasks([waiting_file_id1])
queue2.push_tasks([waiting_file_id2])
- # Mock the Celery group scheduling used by the implementation
- with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
- mock_group.return_value.apply_async = MagicMock()
-
+ # Mock the Celery task publishing used by the implementation
+ with patch.object(Signature, "apply_async", autospec=True) as mock_apply_async:
# Act: Execute the regular task for tenant1 only
rag_pipeline_run_task(file_id1, tenant1.id)
@@ -706,9 +699,9 @@ class TestRagPipelineRunTasks:
assert mock_file_service["delete_file"].call_count == 1
assert mock_pipeline_generator.call_count == 1
- # Verify only tenant1's waiting task was processed (via group)
- assert mock_group.return_value.apply_async.called
- jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+ # Verify only tenant1's waiting task was processed (via apply_async)
+ assert mock_apply_async.called
+ jobs = [call.args[0] for call in mock_apply_async.call_args_list]
first_kwargs = jobs[0].kwargs if jobs else {}
assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id1
assert first_kwargs.get("tenant_id") == tenant1.id
@@ -926,10 +919,8 @@ class TestRagPipelineRunTasks:
waiting_file_id = str(uuid.uuid4())
queue.push_tasks([waiting_file_id])
- # Mock the Celery group scheduling used by the implementation
- with patch("tasks.rag_pipeline.rag_pipeline_run_task.group") as mock_group:
- mock_group.return_value.apply_async = MagicMock()
-
+ # Mock the Celery task publishing used by the implementation
+ with patch.object(Signature, "apply_async", autospec=True) as mock_apply_async:
# Act & Assert: Execute the regular task (should raise Exception)
with pytest.raises(Exception, match="File not found"):
rag_pipeline_run_task(file_id, tenant.id)
@@ -939,10 +930,10 @@ class TestRagPipelineRunTasks:
mock_pipeline_generator.assert_not_called()
# Verify waiting task was still processed despite file error
- assert mock_group.return_value.apply_async.called
+ assert mock_apply_async.called
# Verify correct parameters for the first scheduled job signature
- jobs = mock_group.call_args.args[0] if mock_group.call_args else []
+ jobs = [call.args[0] for call in mock_apply_async.call_args_list]
first_kwargs = jobs[0].kwargs if jobs else {}
assert first_kwargs.get("rag_pipeline_invoke_entities_file_id") == waiting_file_id
assert first_kwargs.get("tenant_id") == tenant.id
diff --git a/api/tests/test_containers_integration_tests/trigger/conftest.py b/api/tests/test_containers_integration_tests/trigger/conftest.py
index 0a8d2a0750d..b6fc7e821ab 100644
--- a/api/tests/test_containers_integration_tests/trigger/conftest.py
+++ b/api/tests/test_containers_integration_tests/trigger/conftest.py
@@ -8,7 +8,6 @@ and mock objects used across trigger-related tests.
from __future__ import annotations
from collections.abc import Generator
-from typing import Any
import pytest
from sqlalchemy import delete
@@ -103,60 +102,14 @@ def app_model(db_session_with_containers: Session, tenant_and_account: tuple[Ten
db_session_with_containers.commit()
-class MockCeleryGroup:
- """Mock for celery group() function that collects dispatched tasks.
-
- Matches the Celery group API loosely, accepting arbitrary kwargs on apply_async
- (e.g. producer) so production code can pass broker-related options without
- breaking tests.
- """
+class RecordingScheduleTask:
+ """Stands in for ``run_schedule_trigger`` and records each dispatched schedule id and producer."""
def __init__(self) -> None:
- self.collected: list[dict[str, Any]] = []
- self._applied = False
- self.last_apply_async_kwargs: dict[str, Any] | None = None
-
- def __call__(self, items: Any) -> MockCeleryGroup:
- self.collected = list(items)
- return self
-
- def apply_async(self, **kwargs: Any) -> None:
- # Accept arbitrary kwargs like producer to be compatible with Celery
- self._applied = True
- self.last_apply_async_kwargs = kwargs
-
- @property
- def applied(self) -> bool:
- return self._applied
-
-
-class MockCelerySignature:
- """Mock for celery task signature that returns task info dict."""
-
- def s(self, schedule_id: str) -> dict[str, str]:
- return {"schedule_id": schedule_id}
-
-
-def mock_celery_group() -> MockCeleryGroup:
- """
- Provide a mock celery group for testing task dispatch.
-
- Returns:
- MockCeleryGroup: Mock group that collects dispatched tasks
- """
- return MockCeleryGroup()
-
+ self.calls: list[tuple[str, object]] = []
-def mock_celery_signature() -> MockCelerySignature:
- """
- Provide a mock celery signature for testing task dispatch.
-
- Returns:
- MockCelerySignature: Mock signature generator
- """
- return MockCelerySignature()
+ def apply_async(self, *, args: tuple[str], producer: object) -> None:
+ self.calls.append((args[0], producer))
class MockPluginSubscription:
diff --git a/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py b/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py
index 7bc578ed6a5..29b784c02e5 100644
--- a/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py
+++ b/api/tests/test_containers_integration_tests/trigger/test_trigger_e2e.py
@@ -47,7 +47,7 @@ from services.trigger.schedule_service import ScheduleService
from services.workflow_service import WorkflowService
from tasks import trigger_processing_tasks
-from .conftest import MockCeleryGroup, MockCelerySignature, MockPluginSubscription
+from .conftest import MockPluginSubscription, RecordingScheduleTask
# Test constants
WEBHOOK_ID_PRODUCTION = "wh1234567890123456789012"
@@ -242,8 +242,6 @@ def test_schedule_poll_dispatches_due_plan(
db_session_with_containers: Session,
tenant_and_account: tuple[Tenant, Account],
app_model: App,
- mock_celery_group: MockCeleryGroup,
- mock_celery_signature: MockCelerySignature,
monkeypatch: pytest.MonkeyPatch,
schedule_type: str,
) -> None:
@@ -271,14 +269,13 @@ def test_schedule_poll_dispatches_due_plan(
next_time = naive_utc_now() + timedelta(hours=1)
monkeypatch.setattr(workflow_schedule_task, "calculate_next_run_at", lambda *_args, **_kwargs: next_time)
- monkeypatch.setattr(workflow_schedule_task, "group", mock_celery_group)
- monkeypatch.setattr(workflow_schedule_task, "run_schedule_trigger", mock_celery_signature)
+ schedule_task = RecordingScheduleTask()
+ monkeypatch.setattr(workflow_schedule_task, "run_schedule_trigger", schedule_task)
poll_workflow_schedules()
- assert mock_celery_group.collected, f"Should dispatch signatures for due {schedule_type} schedules"
- scheduled_ids = {sig["schedule_id"] for sig in mock_celery_group.collected}
- assert plan.id in scheduled_ids
+ assert schedule_task.calls, f"Should dispatch tasks for due {schedule_type} schedules"
+ assert plan.id in {schedule_id for schedule_id, _ in schedule_task.calls}
def test_schedule_visual_debug_poll_generates_event(monkeypatch: pytest.MonkeyPatch) -> None: