Skip to content

armgit / dify

Public Imported from GitHub
0

Build Agentic workflows, RAG pipelines, with rich AI model and tool support on one collaborative workspace. Deploy on cloud, VPC, or self-hosted, so teams move from prototype to production without rebuilding the stack.

fix(api): ignore broken pipe during app stop checks 🤖🤖🤖 (#41356)

QuantumGhost3e12f0b9fb01
diff --git a/api/core/app/apps/base_app_queue_manager.py b/api/core/app/apps/base_app_queue_manager.py
index d8053f2797b..651529ee81d 100644
--- a/api/core/app/apps/base_app_queue_manager.py
+++ b/api/core/app/apps/base_app_queue_manager.py
@@ -1,3 +1,4 @@
+import errno
import logging
import queue
import threading
@@ -30,6 +31,17 @@ from graphon.runtime import GraphRuntimeState
logger = logging.getLogger(__name__)
+def _is_broken_pipe_error(error: BaseException) -> bool:
+ current_error: BaseException | None = error
+ while current_error is not None:
+ if isinstance(current_error, BrokenPipeError):
+ return True
+ if isinstance(current_error, OSError) and current_error.errno == errno.EPIPE:
+ return True
+ current_error = current_error.__cause__ or current_error.__context__
+ return False
+
+
class PublishFrom(IntEnum):
APPLICATION_MANAGER = auto()
TASK_PIPELINE = auto()
@@ -203,12 +215,26 @@ class AppQueueManager(ABC):
@cachedmethod(lambda self: self._stopped_cache, lock=lambda self: self._cache_lock)
def _is_stopped(self) -> bool:
- """
- Check if task is stopped
- :return:
+ """Return whether the task has a stop flag.
+
+ A broken Redis connection cannot establish that a stop was requested,
+ so this check fails open to avoid interrupting the workflow generator.
+ Other Redis errors retain their existing propagation behavior.
"""
stopped_cache_key = AppQueueManager._generate_stopped_cache_key(self._task_id)
- result = redis_client.get(stopped_cache_key)
+ try:
+ result = redis_client.get(stopped_cache_key)
+ except (BrokenPipeError, RedisError) as exc:
+ if not _is_broken_pipe_error(exc):
+ raise
+ logger.warning(
+ "Ignoring broken pipe while checking task stop flag; task=%s key=%s",
+ self._task_id,
+ stopped_cache_key,
+ exc_info=True,
+ )
+ return False
+
if result is not None:
return True
diff --git a/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py b/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py
index 2431b9951ea..4fa04c3da6e 100644
--- a/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py
+++ b/api/tests/unit_tests/core/app/apps/test_base_app_queue_manager.py
@@ -1,7 +1,10 @@
+import errno
+import logging
from types import SimpleNamespace
from unittest.mock import patch
import pytest
+from redis.exceptions import ConnectionError as RedisConnectionError
from core.app.apps.base_app_queue_manager import AppQueueManager, PublishFrom
from core.app.apps.execution_coordinator import AppExecutionState
@@ -19,6 +22,12 @@ class DummyQueueManager(AppQueueManager):
self.published.append((event, pub_from))
+def _redis_broken_pipe_error() -> RedisConnectionError:
+ error = RedisConnectionError("Error 32 while writing to socket. Broken pipe.")
+ error.__context__ = BrokenPipeError(errno.EPIPE, "Broken pipe")
+ return error
+
+
class TestBaseAppQueueManager:
def test_init_requires_user_id(self):
with pytest.raises(ValueError):
@@ -53,6 +62,35 @@ class TestBaseAppQueueManager:
assert manager._is_stopped() is True
+ @pytest.mark.parametrize(
+ "error",
+ [
+ pytest.param(BrokenPipeError(errno.EPIPE, "Broken pipe"), id="direct"),
+ pytest.param(_redis_broken_pipe_error(), id="redis-connection-error"),
+ ],
+ )
+ def test_is_stopped_ignores_broken_pipe(self, error: BaseException, caplog: pytest.LogCaptureFixture):
+ with patch("core.app.apps.base_app_queue_manager.redis_client") as mock_redis:
+ mock_redis.setex.return_value = True
+ mock_redis.get.side_effect = error
+ manager = DummyQueueManager(task_id="t1", user_id="u1", invoke_from=InvokeFrom.SERVICE_API)
+
+ with caplog.at_level(logging.WARNING, logger="core.app.apps.base_app_queue_manager"):
+ assert manager._is_stopped() is False
+
+ assert "Ignoring broken pipe while checking task stop flag" in caplog.text
+ assert "task=t1" in caplog.text
+ assert "key=generate_task_stopped:t1" in caplog.text
+
+ def test_is_stopped_propagates_other_redis_connection_errors(self):
+ with patch("core.app.apps.base_app_queue_manager.redis_client") as mock_redis:
+ mock_redis.setex.return_value = True
+ mock_redis.get.side_effect = RedisConnectionError("Connection refused")
+ manager = DummyQueueManager(task_id="t1", user_id="u1", invoke_from=InvokeFrom.SERVICE_API)
+
+ with pytest.raises(RedisConnectionError, match="Connection refused"):
+ manager._is_stopped()
+
def test_check_for_sqlalchemy_models_raises(self):
with patch("core.app.apps.base_app_queue_manager.redis_client") as mock_redis:
mock_redis.setex.return_value = True