Skip to content

BrainAPI is a knowledge graph–powered AI memory layer that transforms unstructured data into structured knowledge, enabling intelligent search, recommendations, and contextual memory for AI agents and applications.

feat(mcp+worker): fixed long running workers getting stuck + added oauth into mcp

Christianc26b1a75b4fb
diff --git a/.env.example b/.env.example
index ed59576..f7fbd34 100644
--- a/.env.example
+++ b/.env.example
@@ -29,7 +29,13 @@ REDIS_PORT=6379
# Worker envs
# Can be whatever you prefer
# CELERY_BACKEND="rabbitmq"
+# CELERY_UNBLOCKED_WINDOW_SECONDS is the window in seconds during which the number of unblocked events is counted
+# CELERY_UNBLOCKED_MAX_RETRIES is the maximum number of unblocked events that can occur during the window
+# CELERY_QUEUE_HEALTH_THRESHOLD is the maximum number of tasks that can be in the queue before the worker is considered unhealthy
CELERY_BACKEND="redis"
+CELERY_UNBLOCKED_WINDOW_SECONDS=60
+CELERY_UNBLOCKED_MAX_RETRIES=20
+CELERY_QUEUE_HEALTH_THRESHOLD=50
# GraphDB envs
# The following are just examples of the adapter implementation, you can have completely different envs
diff --git a/.gitignore b/.gitignore
index ec827e9..3f828dc 100644
--- a/.gitignore
+++ b/.gitignore
@@ -18,4 +18,6 @@ mcp-stdio-http-bridge/target/
mcp-stdio-http-bridge/dist/
plugins/**
-docs/**
\ No newline at end of file
+docs/**
+
+.git
\ No newline at end of file
diff --git a/README.md b/README.md
index 7b47d06..98fd8f6 100644
--- a/README.md
+++ b/README.md
@@ -1,6 +1,6 @@
<p align="center">
<a href="https://discord.gg/VTngQTaeDf"><img src="https://img.shields.io/badge/Discord-Join%20Lumen%20Brain-5865F2?style=for-the-badge&logo=discord&logoColor=white" alt="Discord"/></a>
- <img src="https://img.shields.io/badge/version-2.11.7--dev-blue?style=for-the-badge" alt="Version"/>
+ <img src="https://img.shields.io/badge/version-2.11.8--dev-blue?style=for-the-badge" alt="Version"/>
<img src="https://img.shields.io/badge/python-3.11+-green?style=for-the-badge&logo=python&logoColor=white" alt="Python"/>
<img src="https://img.shields.io/badge/license-AGPLv3%20%2B%20Commons%20Clause-purple?style=for-the-badge" alt="License"/>
</p>
diff --git a/example-docker-compose.yaml b/example-docker-compose.yaml
index 1a0f5e9..e8a9d2d 100644
--- a/example-docker-compose.yaml
+++ b/example-docker-compose.yaml
@@ -275,6 +275,11 @@ services:
"src.workers.app",
"worker",
"--loglevel=info",
+ "--pool=threads",
+ "--without-gossip",
+ "--without-mingle",
+ "--concurrency=2",
+ "--max-tasks-per-child=50",
]
depends_on:
redis:
@@ -291,11 +296,15 @@ services:
mem_limit: 4g
cpus: 2
healthcheck:
- test: ["CMD", "celery", "-A", "src.workers.app", "inspect", "ping"]
+ test:
+ [
+ "CMD-SHELL",
+ "python -m celery -A src.workers.app inspect ping --timeout 5 >/dev/null 2>&1 && q=$(redis-cli -h redis -p 6379 LLEN ingest_data 2>/dev/null || echo 0); [ \"$q\" -lt \"${CELERY_QUEUE_HEALTH_THRESHOLD:-50}\" ]",
+ ]
interval: 30s
timeout: 10s
retries: 5
- start_period: 900s
+ start_period: 120s
start_interval: 30s
brainapi-mcp:
diff --git a/pyproject.toml b/pyproject.toml
index 71e7acb..abbfe47 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -1,6 +1,6 @@
[project]
name = "brainapi2"
-version = "2.11.7-dev"
+version = "2.11.8-dev"
description = "Version 2.x.x of the BrainAPI memory layer."
authors = [
{name = "Christian",email = "[email protected]"}
diff --git a/src/services/api/app.py b/src/services/api/app.py
index cd50784..dbf6237 100644
--- a/src/services/api/app.py
+++ b/src/services/api/app.py
@@ -106,7 +106,7 @@ def _log_plugin_banner(loader, results: dict[str, bool]):
app = FastAPI(
debug=os.getenv("ENV") == "development",
lifespan=lifespan,
- redirect_slashes=False,
+ redirect_slashes=True,
)
app.add_middleware(BrainPATMiddleware)
diff --git a/src/workers/kombu_redis_patch.py b/src/workers/kombu_redis_patch.py
index 0364b71..ecb7c95 100644
--- a/src/workers/kombu_redis_patch.py
+++ b/src/workers/kombu_redis_patch.py
@@ -1,8 +1,39 @@
from __future__ import annotations
+import logging
+import os
+import threading
+import time
from queue import Empty
_PATCHED = False
+_LOGGER = logging.getLogger(__name__)
+_UNBLOCKED_WINDOW_SECONDS = float(os.getenv("CELERY_UNBLOCKED_WINDOW_SECONDS", "60"))
+_UNBLOCKED_MAX_RETRIES = int(os.getenv("CELERY_UNBLOCKED_MAX_RETRIES", "20"))
+_UNBLOCKED_STATE_LOCK = threading.Lock()
+_UNBLOCKED_WINDOW_STARTED_AT = 0.0
+_UNBLOCKED_EVENT_COUNT = 0
+
+
+def _record_unblocked_event() -> tuple[int, bool]:
+ global _UNBLOCKED_WINDOW_STARTED_AT, _UNBLOCKED_EVENT_COUNT
+
+ now = time.monotonic()
+ with _UNBLOCKED_STATE_LOCK:
+ if (
+ _UNBLOCKED_WINDOW_STARTED_AT == 0.0
+ or now - _UNBLOCKED_WINDOW_STARTED_AT > _UNBLOCKED_WINDOW_SECONDS
+ ):
+ _UNBLOCKED_WINDOW_STARTED_AT = now
+ _UNBLOCKED_EVENT_COUNT = 0
+
+ _UNBLOCKED_EVENT_COUNT += 1
+ current_count = _UNBLOCKED_EVENT_COUNT
+ breached = current_count >= _UNBLOCKED_MAX_RETRIES
+ if breached:
+ _UNBLOCKED_WINDOW_STARTED_AT = 0.0
+ _UNBLOCKED_EVENT_COUNT = 0
+ return current_count, breached
def apply_kombu_redis_unblocked_patch() -> None:
@@ -18,7 +49,20 @@ def apply_kombu_redis_unblocked_patch() -> None:
return _orig_brpop_read(self, **options)
except self.ResponseError as exc:
if str(exc).startswith("UNBLOCKED"):
- self.client.connection.disconnect()
+ try:
+ self.client.connection.disconnect()
+ except Exception:
+ _LOGGER.exception("Celery Redis patch failed to disconnect after UNBLOCKED")
+ event_count, breached = _record_unblocked_event()
+ if event_count == 1:
+ _LOGGER.warning(
+ "Celery Redis consumer received UNBLOCKED; treating as transient and retrying"
+ )
+ if breached:
+ _LOGGER.error(
+ "Celery Redis consumer exceeded UNBLOCKED threshold; re-raising to force restart"
+ )
+ raise
raise Empty() from exc
raise
diff --git a/tests/test_kombu_redis_patch.py b/tests/test_kombu_redis_patch.py
index 1961e1e..40ca0d0 100644
--- a/tests/test_kombu_redis_patch.py
+++ b/tests/test_kombu_redis_patch.py
@@ -14,10 +14,20 @@ def reset_kombu_channel_and_patch_flag():
import src.workers.kombu_redis_patch as patch_mod
saved = Channel._brpop_read
+ saved_monotonic = patch_mod.time.monotonic
patch_mod._PATCHED = False
+ patch_mod._UNBLOCKED_WINDOW_SECONDS = 60
+ patch_mod._UNBLOCKED_MAX_RETRIES = 20
+ patch_mod._UNBLOCKED_WINDOW_STARTED_AT = 0.0
+ patch_mod._UNBLOCKED_EVENT_COUNT = 0
yield
Channel._brpop_read = saved
+ patch_mod.time.monotonic = saved_monotonic
patch_mod._PATCHED = False
+ patch_mod._UNBLOCKED_WINDOW_SECONDS = 60
+ patch_mod._UNBLOCKED_MAX_RETRIES = 20
+ patch_mod._UNBLOCKED_WINDOW_STARTED_AT = 0.0
+ patch_mod._UNBLOCKED_EVENT_COUNT = 0
def test_unblocked_response_error_becomes_empty():
@@ -59,3 +69,94 @@ def test_other_response_error_reraised():
with pytest.raises(ResponseError, match="WRONGTYPE"):
Channel._brpop_read(ch)
ch.client.connection.disconnect.assert_not_called()
+
+
+def test_unblocked_repeats_below_threshold_keep_returning_empty():
+ from kombu.transport.redis import Channel
+
+ import src.workers.kombu_redis_patch as patch_mod
+
+ def fake_brpop(self, **options):
+ raise ResponseError(
+ "UNBLOCKED force unblock from blocking operation, "
+ "instance state changed (master -> replica?)"
+ )
+
+ Channel._brpop_read = fake_brpop
+ patch_mod._UNBLOCKED_MAX_RETRIES = 3
+ apply_kombu_redis_unblocked_patch()
+
+ ch = MagicMock()
+ ch.client.connection.disconnect = MagicMock()
+ ch.connection_errors = ()
+ ch.ResponseError = ResponseError
+
+ with pytest.raises(Empty):
+ Channel._brpop_read(ch)
+ with pytest.raises(Empty):
+ Channel._brpop_read(ch)
+ assert ch.client.connection.disconnect.call_count == 2
+
+
+def test_unblocked_threshold_breaches_and_reraises_response_error():
+ from kombu.transport.redis import Channel
+
+ import src.workers.kombu_redis_patch as patch_mod
+
+ def fake_brpop(self, **options):
+ raise ResponseError(
+ "UNBLOCKED force unblock from blocking operation, "
+ "instance state changed (master -> replica?)"
+ )
+
+ Channel._brpop_read = fake_brpop
+ patch_mod._UNBLOCKED_MAX_RETRIES = 2
+ apply_kombu_redis_unblocked_patch()
+
+ ch = MagicMock()
+ ch.client.connection.disconnect = MagicMock()
+ ch.connection_errors = ()
+ ch.ResponseError = ResponseError
+
+ with pytest.raises(Empty):
+ Channel._brpop_read(ch)
+ with pytest.raises(ResponseError, match="UNBLOCKED"):
+ Channel._brpop_read(ch)
+ assert ch.client.connection.disconnect.call_count == 2
+
+
+def test_unblocked_counter_resets_after_window_expires():
+ from kombu.transport.redis import Channel
+
+ import src.workers.kombu_redis_patch as patch_mod
+
+ class MonotonicSequence:
+ def __init__(self, values):
+ self._it = iter(values)
+
+ def __call__(self):
+ return next(self._it)
+
+ def fake_brpop(self, **options):
+ raise ResponseError(
+ "UNBLOCKED force unblock from blocking operation, "
+ "instance state changed (master -> replica?)"
+ )
+
+ Channel._brpop_read = fake_brpop
+ patch_mod._UNBLOCKED_MAX_RETRIES = 2
+ patch_mod._UNBLOCKED_WINDOW_SECONDS = 5
+ patch_mod.time.monotonic = MonotonicSequence([1.0, 10.0, 10.1])
+ apply_kombu_redis_unblocked_patch()
+
+ ch = MagicMock()
+ ch.client.connection.disconnect = MagicMock()
+ ch.connection_errors = ()
+ ch.ResponseError = ResponseError
+
+ with pytest.raises(Empty):
+ Channel._brpop_read(ch)
+ with pytest.raises(Empty):
+ Channel._brpop_read(ch)
+ with pytest.raises(ResponseError, match="UNBLOCKED"):
+ Channel._brpop_read(ch)