Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): filter benign FalkorDB connection-teardown noise from Sentry (#13678)

Abhimanyu Yadav3e4ac7d13ff5
diff --git a/autogpt_platform/backend/backend/util/metrics.py b/autogpt_platform/backend/backend/util/metrics.py
index a0667804a0..ef60737f2d 100644
--- a/autogpt_platform/backend/backend/util/metrics.py
+++ b/autogpt_platform/backend/backend/util/metrics.py
@@ -1,5 +1,6 @@
import logging
from enum import Enum
+from types import TracebackType
from pydantic import SecretStr
from sentry_sdk._init_implementation import init as _sentry_init
@@ -70,11 +71,40 @@ _PIKA_RECONNECT_SIGNATURES = (
"connection_lost",
)
+# FalkorDB (Graphiti CoPilot memory) connection-teardown noise. graphiti-core's
+# ``execute_query`` logs ERROR + re-raises whenever a query races the closing of
+# its redis.asyncio connection — the fire-and-forget cache-eviction close or a
+# per-request ``driver.close()`` in a finally block. Chat degrades gracefully
+# (memory callers swallow it), so these are benign. Drop only the two teardown
+# signatures, scoped to the graphiti driver (by logger for log events, by
+# traceback for exceptions) so genuine query errors (Cypher typos, missing
+# graphs) — and unrelated main-redis outages — still reach Sentry. (SENTRY-1387.)
+_FALKORDB_DRIVER_LOGGER = "graphiti_core.driver.falkordb_driver"
+_FALKORDB_TEARDOWN_SIGNATURES = (
+ "buffer is closed",
+ "connection closed by server",
+)
+
+
+def _raised_from_graphiti_falkordb(exc_tb: TracebackType | None) -> bool:
+ """True if the graphiti FalkorDB driver module appears in the traceback.
+
+ graphiti-core's ``execute_query`` re-raises from that module, so its frame
+ is on the traceback of a teardown error originating there. Scoping to it
+ keeps unrelated redis exceptions (e.g. the platform's main redis) reaching
+ Sentry even when they share the ``connection closed by server`` message.
+ """
+ while exc_tb is not None:
+ if exc_tb.tb_frame.f_globals.get("__name__") == _FALKORDB_DRIVER_LOGGER:
+ return True
+ exc_tb = exc_tb.tb_next
+ return False
+
def _before_send(event, hint):
"""Filter out expected/transient errors from Sentry to reduce noise."""
if "exc_info" in hint:
- exc_type, exc_value, _ = hint["exc_info"]
+ exc_type, exc_value, exc_tb = hint["exc_info"]
exc_msg = str(exc_value).lower() if exc_value else ""
# AMQP/RabbitMQ transient connection errors — expected during deploys
@@ -91,6 +121,14 @@ def _before_send(event, hint):
) or any(kw in exc_msg for kw in _AMQP_INDICATORS):
return None
+ # FalkorDB connection-teardown races (eviction/shutdown) — benign, but
+ # only when raised from the graphiti driver, so an unrelated main-redis
+ # outage sharing the message still reaches Sentry.
+ if any(
+ sig in exc_msg for sig in _FALKORDB_TEARDOWN_SIGNATURES
+ ) and _raised_from_graphiti_falkordb(exc_tb):
+ return None
+
# User-caused credential/auth/integration errors — not platform bugs
if any(kw in exc_msg for kw in _USER_AUTH_KEYWORDS):
return None
@@ -134,6 +172,9 @@ def _before_send(event, hint):
if logger_name in _PIKA_RECONNECT_LOGGERS and log_msg:
if any(sig in log_msg.lower() for sig in _PIKA_RECONNECT_SIGNATURES):
return None
+ if logger_name == _FALKORDB_DRIVER_LOGGER and log_msg:
+ if any(sig in log_msg.lower() for sig in _FALKORDB_TEARDOWN_SIGNATURES):
+ return None
if logger_name and log_msg:
msg = log_msg.lower()
noisy_log_patterns = [
diff --git a/autogpt_platform/backend/backend/util/metrics_test.py b/autogpt_platform/backend/backend/util/metrics_test.py
index feb5994e31..fa735b42e4 100644
--- a/autogpt_platform/backend/backend/util/metrics_test.py
+++ b/autogpt_platform/backend/backend/util/metrics_test.py
@@ -9,7 +9,11 @@ rather than a Sentry alert flood.
from __future__ import annotations
+import sys
+
from backend.util.metrics import (
+ _FALKORDB_DRIVER_LOGGER,
+ _FALKORDB_TEARDOWN_SIGNATURES,
_PIKA_RECONNECT_LOGGERS,
_PIKA_RECONNECT_SIGNATURES,
_before_send,
@@ -24,6 +28,28 @@ def _log_event(logger: str, message: str) -> dict:
}
+class _RedisConnectionError(Exception):
+ """Exception whose *type* module mimics redis — so tests can tell the old
+ module-based scoping (checked ``exc_type.__module__``) apart from the new
+ traceback-based scoping (checks the raising frame's module)."""
+
+ __module__ = "redis.exceptions"
+
+
+def _exc_info_raised_from(module: str, message: str, exc_type=ConnectionError):
+ """Build an ``exc_info`` triple whose traceback contains a frame in
+ ``module`` and whose exception is ``exc_type`` — lets tests control both
+ the traceback origin (what the new scoping checks) and the exception
+ type's module (what the old module-based scoping checked)."""
+ namespace: dict = {"__name__": module, "_Exc": exc_type}
+ exec("def _raise(msg):\n raise _Exc(msg)", namespace)
+ try:
+ namespace["_raise"](message)
+ except exc_type:
+ return sys.exc_info()
+ raise AssertionError("expected exception")
+
+
# ---------- pika reconnect noise → dropped ----------
@@ -105,3 +131,76 @@ def test_pika_reconnect_signatures_cover_all_four_known_patterns() -> None:
"connection_lost",
}
assert expected == set(_PIKA_RECONNECT_SIGNATURES)
+
+
+# ---------- FalkorDB connection-teardown noise → dropped ----------
+
+
+def test_falkordb_buffer_is_closed_log_dropped() -> None:
+ """SENTRY-1387: ``Buffer is closed`` logged by graphiti-core's FalkorDB
+ driver is a benign connection-teardown race — a query racing the cache
+ eviction close or a per-request ``driver.close()``."""
+ evt = _log_event(
+ _FALKORDB_DRIVER_LOGGER,
+ "Error executing FalkorDB query: Buffer is closed.\nMATCH (n) RETURN n\n{}",
+ )
+ assert _before_send(evt, hint={}) is None
+
+
+def test_falkordb_connection_closed_by_server_log_dropped() -> None:
+ """The sibling teardown message from the same race — the driver docstring
+ pairs it with ``Buffer is closed``."""
+ evt = _log_event(
+ _FALKORDB_DRIVER_LOGGER,
+ "Error executing FalkorDB query: Connection closed by server.",
+ )
+ assert _before_send(evt, hint={}) is None
+
+
+def test_falkordb_buffer_is_closed_exc_from_graphiti_dropped() -> None:
+ """If the re-raised teardown error reaches Sentry as an exception (a caller
+ that doesn't swallow it), a ``Buffer is closed`` raised from the graphiti
+ FalkorDB driver is still benign and must be dropped."""
+ exc_info = _exc_info_raised_from(_FALKORDB_DRIVER_LOGGER, "Buffer is closed.")
+ assert _before_send({"level": "error"}, hint={"exc_info": exc_info}) is None
+
+
+def test_main_redis_connection_closed_exc_kept() -> None:
+ """A genuine ``Connection closed by server`` from the platform's main redis
+ (NOT raised from the graphiti driver) is a real incident and must NOT be
+ swallowed by the teardown-noise rule. Uses a redis-module exception type so
+ the OLD module-based scoping would have dropped it — this test fails if the
+ traceback-based narrowing is reverted."""
+ exc_info = _exc_info_raised_from(
+ "redis.asyncio.connection",
+ "Connection closed by server.",
+ _RedisConnectionError,
+ )
+ assert _before_send({"level": "error"}, hint={"exc_info": exc_info}) is not None
+
+
+def test_falkordb_real_query_error_kept() -> None:
+ """A genuine Cypher/query failure from the same driver logger is load-
+ bearing and must NOT be filtered out by the teardown-noise rule."""
+ evt = _log_event(
+ _FALKORDB_DRIVER_LOGGER,
+ "Error executing FalkorDB query: Invalid input 'RETRUN': expected...",
+ )
+ assert _before_send(evt, hint={}) is not None
+
+
+def test_falkordb_buffer_is_closed_from_other_logger_kept() -> None:
+ """The teardown signatures are only suppressed for the graphiti FalkorDB
+ driver logger; the same string from any other logger is kept."""
+ evt = _log_event(
+ "backend.data.redis_client",
+ "Buffer is closed.",
+ )
+ assert _before_send(evt, hint={}) is not None
+
+
+def test_falkordb_teardown_signatures_cover_known_patterns() -> None:
+ """Sanity check: the teardown-signature list still covers both messages the
+ graphiti FalkorDB driver docstring pairs together."""
+ expected = {"buffer is closed", "connection closed by server"}
+ assert expected == set(_FALKORDB_TEARDOWN_SIGNATURES)