Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): stop FalkorDB driver construction from creating empty graphs (#14052)

Nicholas Tindle601093ddfe23
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/client.py b/autogpt_platform/backend/backend/copilot/graphiti/client.py
index 54d231d849..11efd6cd4c 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/client.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/client.py
@@ -22,7 +22,7 @@ _MAX_GROUP_ID_LEN = 128
# "got Future attached to a different loop". Scope the cache (and its lock)
# per running loop so each loop gets its own clients.
class _LoopState:
- __slots__ = ("cache", "lock")
+ __slots__ = ("cache", "lock", "indexed")
def __init__(self) -> None:
self.cache: TTLCache = _EvictingTTLCache(
@@ -30,6 +30,10 @@ class _LoopState:
ttl=graphiti_config.client_cache_ttl,
)
self.lock = asyncio.Lock()
+ # group_ids whose indices this loop has already ensured. Unbounded
+ # but one short string per group actually *written* to, which is a
+ # far smaller set than the user base — see ``ensure_indices_once``.
+ self.indexed: set[str] = set()
_loop_state: "weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, _LoopState]" = (
@@ -208,6 +212,42 @@ async def get_graphiti_client(group_id: str):
return client
+async def ensure_indices_once(group_id: str, client) -> None:
+ """Build a graph's indices the first time this loop writes to it.
+
+ ``AutoGPTFalkorDriver`` defaults to ``build_indices=False`` so that
+ constructing a driver can never materialize a graph (see its docstring —
+ an init-time ``CREATE INDEX`` is what produced ~13.7k empty graphs in
+ prod). Write paths call this instead: the graph is about to exist
+ anyway, so indexing it is free of that hazard.
+
+ Idempotent and best-effort. Memoized per event loop, so a graph is
+ indexed once per worker rather than on every episode. A failure here
+ must not fail the write, so it is logged and swallowed — the next
+ write for this group retries.
+ """
+ state = _get_loop_state()
+ if group_id in state.indexed:
+ return
+
+ driver = getattr(client, "graph_driver", None) or getattr(client, "driver", None)
+ if driver is None:
+ return
+
+ try:
+ await driver.ensure_indices()
+ except Exception:
+ logger.warning(
+ "Index creation failed for group %s — memory writes continue "
+ "unindexed; will retry on next write",
+ group_id[:16],
+ exc_info=True,
+ )
+ return
+
+ state.indexed.add(group_id)
+
+
async def make_flex_graphiti_client(group_id: str):
"""Return a fresh Graphiti client whose LLM calls run on the flex tier.
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py
index bd4c97c199..811d7fb727 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver.py
@@ -43,19 +43,24 @@ class AutoGPTFalkorDriver(FalkorDriver):
1. ``build_fulltext_query`` adds the per-user ``group_id`` filter so
multi-tenant searches don't cross user graphs.
- 2. ``build_indices`` parameter (defaults True for upstream-compatible
- behaviour) opts out of the fire-and-forget
- ``build_indices_and_constraints`` background task that
- graphiti-core's ``FalkorDriver.__init__`` always spawns.
- That task is fine for long-lived drivers (chat ingest path) but
- generates "Connection closed by server" / "Buffer is closed" log
- spam when the driver is created per short-lived request — most
- notably the admin memory visualizer's per-request driver opens,
- where the indexing task's sequential CREATE INDEX statements
- race the route's own queries and the closing of the connection
- when the route returns. Pass ``build_indices=False`` for
- read-only paths against an existing user's graph; the indices
- are already there from the long-lived chat-write client.
+ 2. ``build_indices`` parameter (defaults **False**) opts out of the
+ fire-and-forget ``build_indices_and_constraints`` background task
+ that graphiti-core's ``FalkorDriver.__init__`` always spawns.
+
+ ``CREATE INDEX`` is a *write*, and in FalkorDB a write
+ **materializes the graph**. So letting that task run on every
+ driver construction mints a fully-indexed, permanently-resident
+ graph for any user we merely *look at* — a search that returns
+ nothing, a warm-context read, a community-rebuild sweep, even the
+ debugging cookbook in this package's AGENTS.md. In prod that
+ produced 13,732 graphs of which ~99.7% held zero nodes, and their
+ index scaffolding (~750KB accounted each) is what pinned FalkorDB
+ at ``maxmemory`` and made it reject every write for 13 days.
+
+ The default is therefore False: constructing a driver must never
+ be able to create a graph. Paths that genuinely write call
+ ``ensure_indices()`` explicitly, which is safe because the graph
+ is about to exist anyway.
3. ``execute_query`` retries FalkorDB's transient "Max pending queries
exceeded" backpressure with bounded jittered backoff, so a load
@@ -63,7 +68,7 @@ class AutoGPTFalkorDriver(FalkorDriver):
dropped one plus a Sentry alert. (SENTRY-1384.)
"""
- def __init__(self, *args, build_indices: bool = True, **kwargs):
+ def __init__(self, *args, build_indices: bool = False, **kwargs):
# Stash the flag BEFORE super().__init__ runs because
# FalkorDriver.__init__ fires
# ``loop.create_task(self.build_indices_and_constraints())``
@@ -73,13 +78,22 @@ class AutoGPTFalkorDriver(FalkorDriver):
super().__init__(*args, **kwargs)
async def build_indices_and_constraints(self) -> None: # type: ignore[override]
- if not getattr(self, "_build_indices_at_init", True):
- # Caller asserted indices already exist (or will be built by
- # someone else) — skip the multi-CREATE-INDEX race that
- # produces the log spam.
+ if not self._build_indices_at_init:
+ # Default path. Suppresses graphiti-core's init-time task so a
+ # bare driver construction cannot materialize an empty graph.
return
await super().build_indices_and_constraints()
+ async def ensure_indices(self) -> None:
+ """Build indices regardless of the ``build_indices`` init flag.
+
+ For write paths only. Bypasses the ``__init__`` suppression above
+ because the caller is about to write — the graph will exist either
+ way, so the indices cost nothing extra and searches over it need
+ them. Callers should invoke this once per graph, not per write.
+ """
+ await super().build_indices_and_constraints()
+
async def execute_query(
self, cypher_query_, **kwargs
) -> tuple[list[dict[str, Any]], list[str], None] | None:
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py
index a46abc0d91..619a39d374 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/falkordb_driver_test.py
@@ -77,9 +77,11 @@ def test_query_without_group_ids_returns_parenthesized_query(
# ---------------------------------------------------------------------------
-# build_indices opt-out — pins the contract that suppresses graphiti-core's
-# per-driver background indexing task on read-only / per-request paths.
-# Regression coverage for the "Buffer is closed" log spam on admin viz loads.
+# build_indices — pins the contract that suppresses graphiti-core's per-driver
+# background indexing task. Default is OFF because CREATE INDEX materializes
+# the graph; only explicit write paths (``ensure_indices``) may create one.
+# Regression coverage for the 13.7k-empty-graph FalkorDB maxmemory wedge, and
+# for the "Buffer is closed" log spam on admin viz loads.
# ---------------------------------------------------------------------------
@@ -116,10 +118,15 @@ async def test_build_indices_true_delegates_to_super() -> None:
@pytest.mark.asyncio
-async def test_default_build_indices_is_upstream_compat() -> None:
- """Omitting the kwarg keeps the upstream-default behaviour so
- existing call sites (long-lived chat client) don't silently lose
- their index-build path."""
+async def test_default_build_indices_does_not_create_graph() -> None:
+ """Omitting the kwarg must NOT build indices.
+
+ ``CREATE INDEX`` is a write, and a write materializes the graph in
+ FalkorDB. An index-building default meant every driver construction —
+ including read-only searches and the AGENTS.md debugging cookbook —
+ minted a permanent empty graph for that user. Prod accumulated 13,732
+ graphs, ~99.7% of them empty, which pinned FalkorDB at maxmemory.
+ """
with patch(
"graphiti_core.driver.falkordb_driver.FalkorDriver.__init__",
return_value=None,
@@ -129,6 +136,24 @@ async def test_default_build_indices_is_upstream_compat() -> None:
) as super_build:
driver = AutoGPTFalkorDriver()
await driver.build_indices_and_constraints()
+ super_build.assert_not_called()
+
+
+async def test_ensure_indices_builds_despite_default_optout() -> None:
+ """``ensure_indices()`` is the write path's explicit opt-in — it must
+ delegate to upstream even though the init-time task is suppressed."""
+ with patch(
+ "graphiti_core.driver.falkordb_driver.FalkorDriver.__init__",
+ return_value=None,
+ ), patch(
+ "graphiti_core.driver.falkordb_driver.FalkorDriver.build_indices_and_constraints",
+ new=AsyncMock(),
+ ) as super_build:
+ driver = AutoGPTFalkorDriver()
+ await driver.build_indices_and_constraints()
+ super_build.assert_not_called()
+ await driver.ensure_indices()
super_build.assert_awaited_once()
diff --git a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py b/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
index ec6c993080..e49bda2792 100644
--- a/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
+++ b/autogpt_platform/backend/backend/copilot/graphiti/ingest.py
@@ -12,7 +12,7 @@ from datetime import datetime, timezone
from graphiti_core.nodes import EpisodeType
-from .client import derive_group_id, get_graphiti_client
+from .client import derive_group_id, ensure_indices_once, get_graphiti_client
from .memory_model import MemoryEnvelope, MemoryKind, MemoryStatus, SourceKind
from .types import EDGE_TYPE_MAP, EDGE_TYPES, ENTITY_TYPES
@@ -259,6 +259,10 @@ async def _ingestion_worker(user_id: str, queue: asyncio.Queue) -> None:
try:
group_id = derive_group_id(user_id)
client = await get_graphiti_client(group_id)
+ # This is the write path, so materializing the graph is
+ # intended here — unlike driver construction, which must
+ # never create one. Once per group per loop.
+ await ensure_indices_once(group_id, client)
# ``_edge_metadata`` is a sidecar (not an add_episode kwarg) —
# pop it before the **payload spread. Present only for dream
# writes; None for conversation turns / memory-store calls.