AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(backend/copilot): retry FalkorDB pending-queue overflow instead of dropping memory ops (#13686)
Abhimanyu Yadav835854c36dda
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/config.py b/autogpt_platform/backend/backend/copilot/graphiti/config.pyindex c9aeb0bf25..b0fab3e4dc 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/config.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/config.py@@ -130,6 +130,28 @@ class GraphitiConfig(BaseSettings):description="Max concurrent LLM calls during ingestion (prevents rate limits)",)+ # FalkorDB back-pressure retry. "Max pending queries exceeded" is a+ # transient, self-clearing rejection when the shared server's pending-query+ # queue is full under concurrent memory traffic; a bounded jittered backoff+ # rides out the spike instead of dropping the memory op (and spamming+ # Sentry). Only an exhausted budget surfaces the error. (SENTRY-1384.)+ falkordb_query_max_attempts: int = Field(+ default=3,+ ge=1,+ description=(+ "Total attempts (including the first) for a FalkorDB query before "+ "surfacing a 'Max pending queries exceeded' backpressure error."+ ),+ )+ falkordb_query_backoff_base: float = Field(+ default=0.1,+ ge=0.0,+ description=(+ "Base delay in seconds for jittered exponential backoff between "+ "FalkorDB pending-queue overflow retries."+ ),+ )+# Warm contextcontext_max_facts: int = Field(default=20)context_timeout: float = Field(diff --git a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.pyindex 55c2f05e7e..bd4c97c199 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py@@ -1,10 +1,44 @@+import asyncio+import logging+import random+from collections.abc import Awaitable+from typing import Any, cast+from graphiti_core.driver.falkordb import STOPWORDSfrom graphiti_core.driver.falkordb_driver import FalkorDriverfrom graphiti_core.helpers import validate_group_ids+from graphiti_core.utils.datetime_utils import convert_datetimes_to_strings++from .config import graphiti_config++logger = logging.getLogger(__name__)++# graphiti-core's ``FalkorDriver.execute_query`` logs terminal query errors+# under this logger name. Our ``execute_query`` override no longer delegates to+# ``super()`` (so it can keep intermediate retries silent), so it re-emits+# terminal errors under the SAME logger to preserve Sentry grouping — and to+# keep the SENTRY-1387 benign-teardown filter, which keys on this logger,+# working on the query path.+_UPSTREAM_QUERY_LOGGER = logging.getLogger("graphiti_core.driver.falkordb_driver")++# FalkorDB sheds load with this message when its global pending-query queue is+# full. It is a *pre-execution* rejection — the query never ran — so retrying+# is side-effect-free and safe even for writes.+_PENDING_QUEUE_OVERFLOW = "max pending queries exceeded"++# Cap each retry's total wait (jitter included) so a mis-tuned high+# ``falkordb_query_max_attempts`` / ``falkordb_query_backoff_base`` can't balloon+# one wait into minutes. At default settings total backoff stays well within the+# warm-context timeout budget.+_MAX_RETRY_DELAY_SECONDS = 2.0+++def _is_pending_queue_overflow(exc: Exception) -> bool:+ return _PENDING_QUEUE_OVERFLOW in str(exc).lower()class AutoGPTFalkorDriver(FalkorDriver):- """FalkorDriver subclass with two AutoGPT-specific tweaks.+ """FalkorDriver subclass with three AutoGPT-specific tweaks.1. ``build_fulltext_query`` adds the per-user ``group_id`` filter somulti-tenant searches don't cross user graphs.@@ -22,6 +56,11 @@ class AutoGPTFalkorDriver(FalkorDriver):when the route returns. Pass ``build_indices=False`` forread-only paths against an existing user's graph; the indicesare already there from the long-lived chat-write client.++ 3. ``execute_query`` retries FalkorDB's transient "Max pending queries+ exceeded" backpressure with bounded jittered backoff, so a load+ spike degrades into a slightly slower memory op instead of a+ dropped one plus a Sentry alert. (SENTRY-1384.)"""def __init__(self, *args, build_indices: bool = True, **kwargs):@@ -41,6 +80,76 @@ class AutoGPTFalkorDriver(FalkorDriver):returnawait super().build_indices_and_constraints()+ async def execute_query(+ self, cypher_query_, **kwargs+ ) -> tuple[list[dict[str, Any]], list[str], None] | None:+ """Run a Cypher query, retrying transient pending-queue overflow.++ "Max pending queries exceeded" is FalkorDB backpressure: the shared+ server's pending-query queue is full under concurrent memory traffic+ (ingest fan-out, warm-context searches, community rebuilds). It clears+ within milliseconds, so the rejected query is retried with jittered+ exponential backoff. Only an exhausted retry budget reaches Sentry —+ intermediate attempts stay silent. Every other error (Cypher typo,+ missing graph, connection teardown) fails fast on the first attempt,+ exactly as upstream ``FalkorDriver.execute_query``.++ Reimplemented rather than delegating to ``super()`` so per-attempt+ logging is under our control; the result-shaping mirrors upstream.+ """+ graph = self._get_graph(self._database)+ # falkordb's async ``Graph.query`` is typed as its sync counterpart in+ # the stubs (returns a bare ``QueryResult``, params typed narrowly);+ # cast to the real awaitable/param types rather than suppress. Same+ # runtime call upstream ``FalkorDriver.execute_query`` makes.+ params = cast(dict[str, object], convert_datetimes_to_strings(dict(kwargs)))+ attempts = max(1, graphiti_config.falkordb_query_max_attempts)++ for attempt in range(attempts):+ try:+ result = await cast(Awaitable[Any], graph.query(cypher_query_, params))+ return self._to_records(result)+ except Exception as e:+ if "already indexed" in str(e):+ _UPSTREAM_QUERY_LOGGER.info(f"Index already exists: {e}")+ return None+ if _is_pending_queue_overflow(e) and attempt < attempts - 1:+ delay = self._pending_queue_retry_delay(attempt)+ logger.debug(+ "FalkorDB pending-queue overflow; retry %s/%s in %.2fs",+ attempt + 1,+ attempts,+ delay,+ )+ await asyncio.sleep(delay)+ continue+ # ``params`` hold user memory content (names, facts) — omit them+ # from this Sentry-routed log to avoid leaking PII. The exception+ # and query text are enough to triage a genuine failure.+ _UPSTREAM_QUERY_LOGGER.error(+ f"Error executing FalkorDB query: {e}\n{cypher_query_}"+ )+ raise+ raise AssertionError("unreachable: loop returns or raises on every path")++ @staticmethod+ def _pending_queue_retry_delay(attempt: int) -> float:+ """Jittered exponential backoff (capped) so retries drain the queue over+ time and don't synchronize across concurrent callers."""+ base = graphiti_config.falkordb_query_backoff_base+ delay = base * (2**attempt) + random.uniform(0, base)+ return min(delay, _MAX_RETRY_DELAY_SECONDS)++ @staticmethod+ def _to_records(result) -> tuple[list[dict[str, Any]], list[str], None]:+ """Mirror of upstream's list-of-lists → list-of-dicts result shaping."""+ header = [h[1] for h in result.header]+ records = [+ {name: (row[i] if i < len(row) else None) for i, name in enumerate(header)}+ for row in result.result_set+ ]+ return records, header, None+def build_fulltext_query(self,query: str,diff --git a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.pyindex eed7e5b60a..a46abc0d91 100644--- a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py+++ b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py@@ -2,9 +2,29 @@ from unittest.mock import AsyncMock, MagicMock, patchimport pytest+from . import falkordb_driver as fdbfrom .falkordb_driver import AutoGPTFalkorDriver+class _FakeResult:+ """Minimal stand-in for a falkordb query result (header + rows)."""++ def __init__(self, header, result_set):+ self.header = header+ self.result_set = result_set+++def _set_query(driver: AutoGPTFalkorDriver, side_effect) -> AsyncMock:+ """Wire ``graph.query`` (reached via ``_get_graph``) to ``side_effect``."""+ query = AsyncMock(side_effect=side_effect)+ driver.client.select_graph.return_value.query = query+ return query+++def _overflow() -> Exception:+ return Exception("Max pending queries exceeded")++@pytest.fixturedef driver() -> AutoGPTFalkorDriver:# ``build_fulltext_query`` is a pure string-builder that never touches@@ -128,3 +148,126 @@ async def test_build_indices_false_persists_across_repeated_calls() -> None:await driver.build_indices_and_constraints()await driver.build_indices_and_constraints()super_build.assert_not_called()+++# ---------------------------------------------------------------------------+# execute_query retry — pins the bounded backoff on FalkorDB's transient+# "Max pending queries exceeded" backpressure. (SENTRY-1384.)+# ---------------------------------------------------------------------------+++async def test_execute_query_success_returns_upstream_shaped_records(+ driver: AutoGPTFalkorDriver,+) -> None:+ """Happy path: no retry, and results are shaped like upstream+ (list-of-dicts, header, None)."""+ _set_query(driver, [_FakeResult([("n.count", "count")], [[5]])])++ records, header, meta = await driver.execute_query("MATCH (n) RETURN count(n)")++ assert records == [{"count": 5}]+ assert header == ["count"]+ assert meta is None+++async def test_execute_query_retries_pending_queue_overflow_then_succeeds(+ driver: AutoGPTFalkorDriver,+) -> None:+ """Two transient overflows are retried; the third attempt succeeds and its+ result is returned (memory op recovers instead of being dropped)."""+ query = _set_query(+ driver,+ [_overflow(), _overflow(), _FakeResult([("x", "count")], [[1]])],+ )++ # max_attempts=5 (non-default) so the third attempt succeeds well within+ # budget — proves retries continue past the first failure.+ with patch.object(fdb.asyncio, "sleep", new=AsyncMock()) as sleep, patch(+ "backend.copilot.graphiti.config.graphiti_config.falkordb_query_max_attempts",+ 5,+ ):+ records, _, _ = await driver.execute_query("MATCH (n) RETURN 1")++ assert records == [{"count": 1}]+ assert query.await_count == 3+ assert sleep.await_count == 2 # backoff between the three attempts+++async def test_execute_query_raises_after_exhausting_retries(+ driver: AutoGPTFalkorDriver,+) -> None:+ """A sustained overflow exhausts the budget, then raises AND logs exactly+ one terminal error under the upstream logger (so Sentry sees one event)."""+ query = _set_query(driver, _overflow())++ # max_attempts=4 (non-default) proves the knob controls the attempt count.+ with patch.object(fdb.asyncio, "sleep", new=AsyncMock()) as sleep, patch.object(+ fdb, "_UPSTREAM_QUERY_LOGGER"+ ) as upstream_logger, patch(+ "backend.copilot.graphiti.config.graphiti_config.falkordb_query_max_attempts",+ 4,+ ):+ with pytest.raises(Exception, match="Max pending queries exceeded"):+ await driver.execute_query("MATCH (n) RETURN 1")++ assert query.await_count == 4+ assert sleep.await_count == 3+ upstream_logger.error.assert_called_once()+++async def test_pending_queue_retry_delay_is_bounded_and_jittered() -> None:+ """Backoff grows exponentially within [base*2**n, base*2**n + base) and is+ capped so a high max_attempts can't balloon a single wait."""+ with patch(+ "backend.copilot.graphiti.config.graphiti_config.falkordb_query_backoff_base",+ 0.1,+ ):+ assert 0.1 <= AutoGPTFalkorDriver._pending_queue_retry_delay(0) < 0.2+ assert 0.2 <= AutoGPTFalkorDriver._pending_queue_retry_delay(1) < 0.3+ # attempt 20 would be ~100k*base uncapped; the cap holds it near the max.+ capped = AutoGPTFalkorDriver._pending_queue_retry_delay(20)+ assert (+ fdb._MAX_RETRY_DELAY_SECONDS <= capped < fdb._MAX_RETRY_DELAY_SECONDS + 0.1+ )+++async def test_execute_query_non_overflow_error_fails_fast(+ driver: AutoGPTFalkorDriver,+) -> None:+ """A genuine query error (Cypher typo, missing graph, teardown) is not+ retried — it raises on the first attempt and logs once."""+ query = _set_query(driver, ValueError("syntax error near RETRN"))++ with patch.object(fdb.asyncio, "sleep", new=AsyncMock()) as sleep, patch.object(+ fdb, "_UPSTREAM_QUERY_LOGGER"+ ) as upstream_logger:+ with pytest.raises(ValueError):+ await driver.execute_query("MATCH (n) RETRN 1")++ assert query.await_count == 1+ sleep.assert_not_awaited()+ upstream_logger.error.assert_called_once()+++async def test_execute_query_already_indexed_returns_none_without_retry(+ driver: AutoGPTFalkorDriver,+) -> None:+ """The upstream 'already indexed' short-circuit is preserved: returns None,+ no retry, no terminal error."""+ query = _set_query(driver, Exception("Index already indexed"))++ with patch.object(fdb.asyncio, "sleep", new=AsyncMock()) as sleep, patch.object(+ fdb, "_UPSTREAM_QUERY_LOGGER"+ ) as upstream_logger:+ result = await driver.execute_query("CREATE INDEX ...")++ assert result is None+ assert query.await_count == 1+ sleep.assert_not_awaited()+ upstream_logger.error.assert_not_called()