Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

fix(celery): recover from Redis UNBLOCKED during BRPOP failover

Cursor Agente2f431472b20
diff --git a/src/workers/app.py b/src/workers/app.py
index b79db0c..fdaf412 100644
--- a/src/workers/app.py
+++ b/src/workers/app.py
@@ -23,6 +23,10 @@ warnings.filterwarnings(
category=UserWarning,
)
+from src.workers.kombu_redis_patch import apply_kombu_redis_unblocked_patch
+
+apply_kombu_redis_unblocked_patch()
+
from celery import Celery
from kombu import Queue
diff --git a/src/workers/kombu_redis_patch.py b/src/workers/kombu_redis_patch.py
new file mode 100644
index 0000000..0364b71
--- /dev/null
+++ b/src/workers/kombu_redis_patch.py
@@ -0,0 +1,26 @@
+from __future__ import annotations
+
+from queue import Empty
+
+_PATCHED = False
+
+
+def apply_kombu_redis_unblocked_patch() -> None:
+ global _PATCHED
+ if _PATCHED:
+ return
+ from kombu.transport.redis import Channel
+
+ _orig_brpop_read = Channel._brpop_read
+
+ def _brpop_read(self, **options):
+ try:
+ return _orig_brpop_read(self, **options)
+ except self.ResponseError as exc:
+ if str(exc).startswith("UNBLOCKED"):
+ self.client.connection.disconnect()
+ raise Empty() from exc
+ raise
+
+ Channel._brpop_read = _brpop_read
+ _PATCHED = True
diff --git a/tests/test_kombu_redis_patch.py b/tests/test_kombu_redis_patch.py
new file mode 100644
index 0000000..1961e1e
--- /dev/null
+++ b/tests/test_kombu_redis_patch.py
@@ -0,0 +1,61 @@
+from queue import Empty
+from unittest.mock import MagicMock
+
+import pytest
+from redis.exceptions import ResponseError
+
+from src.workers.kombu_redis_patch import apply_kombu_redis_unblocked_patch
+
+
[email protected](autouse=True)
+def reset_kombu_channel_and_patch_flag():
+ from kombu.transport.redis import Channel
+
+ import src.workers.kombu_redis_patch as patch_mod
+
+ saved = Channel._brpop_read
+ patch_mod._PATCHED = False
+ yield
+ Channel._brpop_read = saved
+ patch_mod._PATCHED = False
+
+
+def test_unblocked_response_error_becomes_empty():
+ from kombu.transport.redis import Channel
+
+ def fake_brpop(self, **options):
+ raise ResponseError(
+ "UNBLOCKED force unblock from blocking operation, "
+ "instance state changed (master -> replica?)"
+ )
+
+ Channel._brpop_read = fake_brpop
+ apply_kombu_redis_unblocked_patch()
+
+ ch = MagicMock()
+ ch.client.connection.disconnect = MagicMock()
+ ch.connection_errors = ()
+ ch.ResponseError = ResponseError
+
+ with pytest.raises(Empty):
+ Channel._brpop_read(ch)
+ ch.client.connection.disconnect.assert_called_once()
+
+
+def test_other_response_error_reraised():
+ from kombu.transport.redis import Channel
+
+ def fake_brpop(self, **options):
+ raise ResponseError("WRONGTYPE")
+
+ Channel._brpop_read = fake_brpop
+ apply_kombu_redis_unblocked_patch()
+
+ ch = MagicMock()
+ ch.client.connection.disconnect = MagicMock()
+ ch.connection_errors = ()
+ ch.ResponseError = ResponseError
+
+ with pytest.raises(ResponseError, match="WRONGTYPE"):
+ Channel._brpop_read(ch)
+ ch.client.connection.disconnect.assert_not_called()