Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): share PostHog flag definitions through one refresher and a Redis cache (#14767)

Reinier van der Leer362e7522ff66
diff --git a/autogpt_platform/backend/.env.default b/autogpt_platform/backend/.env.default
index bc18998508..bd9b73d97a 100644
--- a/autogpt_platform/backend/.env.default
+++ b/autogpt_platform/backend/.env.default
@@ -358,6 +358,13 @@ POSTHOG_HOST=https://eu.i.posthog.com
# Personal API key, only for local feature-flag evaluation (Settings > Personal
# API keys, scope: feature_flag:read). Without it flag reads go over the wire.
POSTHOG_PERSONAL_API_KEY=
+# Where flag definitions are shared between processes: redis (default; one
+# elected refresher, every other process reads its copy), memory (process-local),
+# or none (every process polls PostHog itself). PostHog bills one definitions
+# fetch as ten flag requests, so "none" scales that bill with replica count.
+POSTHOG_FLAG_DEFINITION_CACHE=redis
+POSTHOG_FLAG_DEFINITION_REFRESH_SECONDS=30
+POSTHOG_FLAG_DEFINITION_CACHE_TTL_SECONDS=600
# Tally Form Integration (pre-populate business understanding on signup)
TALLY_API_KEY=
diff --git a/autogpt_platform/backend/backend/data/redis_client.py b/autogpt_platform/backend/backend/data/redis_client.py
index 63ac49d857..fcd5fb388c 100644
--- a/autogpt_platform/backend/backend/data/redis_client.py
+++ b/autogpt_platform/backend/backend/data/redis_client.py
@@ -10,7 +10,7 @@ from redis.asyncio import Redis as AsyncRedis
from redis.asyncio.cluster import ClusterNode as AsyncClusterNode
from redis.asyncio.cluster import RedisCluster as AsyncRedisCluster
from redis.asyncio.retry import Retry as AsyncRetry
-from redis.backoff import ExponentialBackoff
+from redis.backoff import ExponentialBackoff, NoBackoff
from redis.cluster import ClusterNode, RedisCluster
from redis.exceptions import ClusterDownError
from redis.exceptions import ConnectionError as RedisConnectionError
@@ -119,6 +119,30 @@ def connect() -> RedisClient:
return c
+def connect_once(timeout: float) -> RedisClient:
+ """One attempt, short timeouts, no retries: for callers with a fallback of their own."""
+ c = RedisCluster(
+ startup_nodes=[ClusterNode(HOST, PORT)],
+ password=PASSWORD,
+ decode_responses=True,
+ socket_timeout=timeout,
+ socket_connect_timeout=timeout,
+ socket_keepalive=True,
+ health_check_interval=HEALTH_CHECK_INTERVAL,
+ address_remap=_address_remap,
+ retry=Retry(NoBackoff(), 0),
+ )
+ try:
+ c.ping()
+ except Exception:
+ try:
+ c.close()
+ except Exception:
+ pass
+ raise
+ return c
+
+
@conn_retry("Redis", "Releasing connection")
def disconnect():
get_redis().close()
diff --git a/autogpt_platform/backend/backend/util/feature_flag/definition_cache.py b/autogpt_platform/backend/backend/util/feature_flag/definition_cache.py
new file mode 100644
index 0000000000..fb4fe23cb8
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/feature_flag/definition_cache.py
@@ -0,0 +1,294 @@
+"""Shared cache for PostHog's flag definitions.
+
+Local evaluation makes every process poll PostHog for the flag definitions, and
+PostHog bills one definitions fetch as ten flag requests, so the bill scales
+with replica count. Here one elected process refreshes the definitions and
+writes them to Redis; every other process reads that copy.
+
+Implements PostHog's own ``FlagDefinitionCacheProvider`` protocol, which the
+SDK calls only from its poller thread — never from a flag read, so no
+evaluation can block on Redis.
+"""
+
+import json
+import logging
+import os
+import socket
+import threading
+import time
+import uuid
+from typing import Any, Callable
+
+from posthog.flag_definition_cache import (
+ FlagDefinitionCacheData,
+ FlagDefinitionCacheProvider,
+)
+from prometheus_client import Counter
+
+from backend.data.redis_client import connect_once
+from backend.util.settings import FlagDefinitionCacheBackend, Settings
+
+logger = logging.getLogger(__name__)
+
+settings = Settings()
+
+# Not in backend.monitoring.instrumentation with its siblings: that package
+# imports backend.util.metrics, which imports the flag module this one serves.
+# The default registry is what /metrics renders, wherever a counter is declared.
+CACHE_EVENTS = Counter(
+ "autogpt_posthog_flag_definition_cache_events_total",
+ "PostHog flag-definition cache events, by outcome",
+ # refresher / follower / cached / stale / empty / stored / error — bounded
+ labelnames=["outcome"],
+)
+
+_DATA_KEY = "posthog:flag_definitions"
+_LOCK_KEY = "posthog:flag_definitions:refresher"
+
+# The refresher renews its lock every poll, so a lock outliving two polls hands
+# the job to another process within ~2 intervals of the refresher dying.
+_LOCK_TTL_POLLS = 2
+# Past this the refresher is late or gone. Stale definitions still beat none.
+_STALE_AFTER_POLLS = 5
+# One line per poll per process would drown the logs for as long as Redis is out.
+_DEGRADED_LOG_INTERVAL = 300.0
+# The SDK's poller thread waits on these; past them it falls back to fetching.
+_REDIS_TIMEOUT_SECONDS = 2.0
+
+# Renew and release only our own lock: a bare PEXPIRE or DEL would extend or
+# free whichever process actually holds it.
+_RENEW_LOCK = """
+if redis.call('get', KEYS[1]) == ARGV[1] then
+ return redis.call('pexpire', KEYS[1], ARGV[2])
+end
+return 0
+"""
+
+_RELEASE_LOCK = """
+if redis.call('get', KEYS[1]) == ARGV[1] then
+ return redis.call('del', KEYS[1])
+end
+return 0
+"""
+
+
+def get_flag_definition_cache() -> FlagDefinitionCacheProvider | None:
+ """The provider the PostHog client shares its flag definitions through.
+
+ None leaves the SDK polling PostHog once per process, which is what it does
+ without a provider.
+ """
+ backend = settings.config.posthog_flag_definition_cache
+ if backend is FlagDefinitionCacheBackend.NONE:
+ return None
+ if backend is FlagDefinitionCacheBackend.MEMORY:
+ return MemoryFlagDefinitionCache(refresh_interval=refresh_interval_seconds())
+ return RedisFlagDefinitionCache(
+ refresh_interval=refresh_interval_seconds(),
+ ttl=settings.config.posthog_flag_definition_cache_ttl_seconds,
+ )
+
+
+def refresh_interval_seconds() -> int:
+ """How often the SDK polls: the refresher fetches, the rest re-read."""
+ return settings.config.posthog_flag_definition_refresh_seconds
+
+
+class RedisFlagDefinitionCache:
+ """Elects one refresher through a Redis lock; the rest read its copy.
+
+ Every failure here degrades to the behaviour we had without it — a direct
+ fetch, or the definitions already in memory — and never raises at a caller.
+ """
+
+ def __init__(
+ self,
+ *,
+ refresh_interval: int,
+ ttl: int,
+ redis_factory: Callable[[], Any] | None = None,
+ ) -> None:
+ self._ttl = ttl
+ self._lock_ttl_ms = refresh_interval * _LOCK_TTL_POLLS * 1000
+ self._stale_after = refresh_interval * _STALE_AFTER_POLLS
+ self._redis_factory = redis_factory or _connect
+ self._client: Any = None
+ self._instance = f"{socket.gethostname()}:{os.getpid()}:{uuid.uuid4().hex[:8]}"
+ self._is_refresher = False
+ self._degraded_at = 0.0
+ # What this process's SDK last installed, fetched or read.
+ self._definitions: FlagDefinitionCacheData | None = None
+
+ def should_fetch_flag_definitions(self) -> bool:
+ try:
+ redis = self._redis()
+ acquired = bool(
+ redis.set(_LOCK_KEY, self._instance, nx=True, px=self._lock_ttl_ms)
+ ) or bool(
+ redis.eval(_RENEW_LOCK, 1, _LOCK_KEY, self._instance, self._lock_ttl_ms)
+ )
+ # The SDK stores only on a 200; a 304 poll would otherwise let the
+ # shared copy lapse while the definitions are unchanged.
+ if acquired and self._definitions is not None:
+ self._store(redis, self._definitions)
+ except Exception as e:
+ self._drop_client()
+ self._log_degraded("electing a flag-definition refresher", e)
+ _record("error")
+ # What the SDK falls back to on a provider error anyway: definitions
+ # fetched per process beat definitions nobody refreshes.
+ return True
+
+ self._note_role(acquired)
+ _record("refresher" if acquired else "follower")
+ return acquired
+
+ def get_flag_definitions(self) -> FlagDefinitionCacheData | None:
+ try:
+ raw = self._redis().get(_DATA_KEY)
+ except Exception as e:
+ self._drop_client()
+ self._log_degraded("reading the shared flag definitions", e)
+ _record("error")
+ # None leaves the SDK on the definitions it already holds, and makes
+ # it fetch directly only when it has none at all.
+ return None
+
+ if raw is None:
+ _record("empty")
+ return None
+
+ try:
+ cached = json.loads(raw)
+ data: FlagDefinitionCacheData = cached["definitions"]
+ age = time.time() - float(cached["fetched_at"])
+ except (TypeError, ValueError, KeyError) as e:
+ logger.warning(f"Unreadable shared PostHog flag definitions: {e}")
+ _record("error")
+ return None
+
+ if age > self._stale_after:
+ logger.warning(
+ f"Shared PostHog flag definitions are {age:.0f}s old; serving them "
+ "while a refresher catches up"
+ )
+ _record("stale")
+ else:
+ _record("cached")
+ self._definitions = data
+ return data
+
+ def on_flag_definitions_received(self, data: FlagDefinitionCacheData) -> None:
+ self._definitions = data
+ try:
+ self._store(self._redis(), data)
+ except Exception as e:
+ self._drop_client()
+ self._log_degraded("sharing the refreshed flag definitions", e)
+ _record("error")
+ return
+
+ logger.debug(f"Shared {len(data.get('flags') or [])} PostHog flag definitions")
+
+ def shutdown(self) -> None:
+ # Connecting here would hold shutdown for a Redis that is down anyway;
+ # the lock then lapses on its own within two polls.
+ if not self._is_refresher or self._client is None:
+ return
+ try:
+ self._client.eval(_RELEASE_LOCK, 1, _LOCK_KEY, self._instance)
+ except Exception as e:
+ logger.warning(f"Could not release the flag-refresher lock: {e}")
+ finally:
+ self._is_refresher = False
+ self._drop_client()
+
+ def _redis(self) -> Any:
+ if self._client is None:
+ self._client = self._redis_factory()
+ return self._client
+
+ def _drop_client(self) -> None:
+ client, self._client = self._client, None
+ if client is None:
+ return
+ try:
+ client.close()
+ except Exception:
+ pass
+
+ def _store(self, redis: Any, data: FlagDefinitionCacheData) -> None:
+ payload = json.dumps({"fetched_at": time.time(), "definitions": data})
+ redis.set(_DATA_KEY, payload, ex=self._ttl)
+ _record("stored")
+
+ def _note_role(self, is_refresher: bool) -> None:
+ if is_refresher == self._is_refresher:
+ return
+ self._is_refresher = is_refresher
+ logger.info(
+ f"This process now refreshes the shared PostHog flag definitions "
+ f"({self._instance})"
+ if is_refresher
+ else "Another process now refreshes the shared PostHog flag definitions"
+ )
+
+ def _log_degraded(self, what: str, error: Exception) -> None:
+ now = time.monotonic()
+ if self._degraded_at and now - self._degraded_at < _DEGRADED_LOG_INTERVAL:
+ return
+ self._degraded_at = now
+ logger.warning(
+ f"Redis unavailable while {what}: {error}. Each process polls PostHog "
+ "for itself until it is back."
+ )
+
+
+class MemoryFlagDefinitionCache:
+ """Process-local stand-in, for local dev and tests.
+
+ One process is trivially its own refresher; this exists to exercise the
+ provider path without Redis.
+ """
+
+ def __init__(self, *, refresh_interval: int) -> None:
+ self._refresh_interval = refresh_interval
+ self._data: FlagDefinitionCacheData | None = None
+ self._fetched_at = 0.0
+ self._lock = threading.Lock()
+
+ def should_fetch_flag_definitions(self) -> bool:
+ with self._lock:
+ fresh = (
+ self._data is not None
+ and time.monotonic() - self._fetched_at < self._refresh_interval
+ )
+ _record("follower" if fresh else "refresher")
+ return not fresh
+
+ def get_flag_definitions(self) -> FlagDefinitionCacheData | None:
+ with self._lock:
+ data = self._data
+ _record("cached" if data else "empty")
+ return data
+
+ def on_flag_definitions_received(self, data: FlagDefinitionCacheData) -> None:
+ with self._lock:
+ self._data = data
+ self._fetched_at = time.monotonic()
+ _record("stored")
+
+ def shutdown(self) -> None:
+ with self._lock:
+ self._data = None
+
+
+def _record(outcome: str) -> None:
+ """In a healthy fleet one process records ``stored`` and the rest ``cached``."""
+ CACHE_EVENTS.labels(outcome=outcome).inc()
+
+
+def _connect() -> Any:
+ # Not the shared get_redis(): its connect retries for minutes, and this
+ # cache has a fallback of its own.
+ return connect_once(timeout=_REDIS_TIMEOUT_SECONDS)
diff --git a/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py b/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py
new file mode 100644
index 0000000000..3dcd7227b0
--- /dev/null
+++ b/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py
@@ -0,0 +1,566 @@
+"""The shared flag-definition cache: election, failure modes, and the SDK wiring."""
+
+import json
+import logging
+import time
+import uuid
+
+import pytest
+from ldclient import Context, LDClient
+from posthog import Posthog
+from posthog.request import GetResponse
+from pydantic import ValidationError
+
+import backend.data.redis_client as redis_client
+import backend.util.feature_flag as ff
+import backend.util.feature_flag.definition_cache as cache
+import backend.util.feature_flag.posthog as ph
+from backend.util.feature_flag import Flag, evaluate_feature_flag
+from backend.util.settings import Config, FeatureFlagBackend, FlagDefinitionCacheBackend
+from backend.util.testing import is_tcp_port_reachable
+
+REFRESH = 30
+LOCK_TTL = REFRESH * cache._LOCK_TTL_POLLS
+
+
+class FakeRedis:
+ """The four commands the provider uses, over a clock the test controls."""
+
+ def __init__(self):
+ self.now = 1_000.0
+ self.fail = False
+ self._values: dict[str, tuple[str, float | None]] = {}
+
+ def set(self, key, value, nx=False, px=None, ex=None):
+ self._check()
+ if nx and key in self._values:
+ return None
+ ttl = px / 1000 if px else (float(ex) if ex else None)
+ self._values[key] = (value, self.now + ttl if ttl else None)
+ return True
+
+ def get(self, key):
+ self._check()
+ entry = self._values.get(key)
+ return entry[0] if entry else None
+
+ def eval(self, script, numkeys, key, holder, *args):
+ self._check()
+ entry = self._values.get(key)
+ if not entry or entry[0] != holder:
+ return 0
+ if script is cache._RENEW_LOCK:
+ self._values[key] = (entry[0], self.now + int(args[0]) / 1000)
+ return 1
+ assert script is cache._RELEASE_LOCK
+ del self._values[key]
+ return 1
+
+ def advance(self, seconds: float):
+ self.now += seconds
+
+ def _check(self):
+ if self.fail:
+ raise ConnectionError("redis is down")
+ for key, (_, expires) in list(self._values.items()):
+ if expires is not None and expires <= self.now:
+ del self._values[key]
+
+
+DEFINITIONS = {
+ "flags": [
+ {
+ "id": 1,
+ "team_id": 1,
+ "name": "",
+ "key": "shared-flag",
+ "active": True,
+ "deleted": False,
+ "ensure_experience_continuity": False,
+ "filters": {"groups": [{"properties": [], "rollout_percentage": 100}]},
+ }
+ ],
+ "group_type_mapping": {},
+ "cohorts": {},
+}
+
+
+def redis():
+ return FakeRedis()
+
+
+def logs():
+ """`caplog` captures nothing for `backend.*` under the app's logging config."""
+ captured: list[str] = []
+
+ class Capture(logging.Handler):
+ def emit(self, record):
+ captured.append(record.getMessage())
+
+ handler = Capture()
+ cache.logger.addHandler(handler)
+ previous, cache.logger.level = cache.logger.level, logging.DEBUG
+ yield captured
+ cache.logger.removeHandler(handler)
+ cache.logger.level = previous
+
+
+def provider(redis, *, ttl=600) -> cache.RedisFlagDefinitionCache:
+ return cache.RedisFlagDefinitionCache(
+ refresh_interval=REFRESH, ttl=ttl, redis_factory=lambda: redis
+ )
+
+
+def store_definitions(redis, *, age_seconds=0.0, definitions=None):
+ """Write the shared copy as the refresher would have, `age_seconds` ago."""
+ redis.set(
+ cache._DATA_KEY,
+ json.dumps(
+ {
+ "fetched_at": time.time() - age_seconds,
+ "definitions": definitions if definitions is not None else DEFINITIONS,
+ }
+ ),
+ )
+
+
+class TestElection:
+ def test_exactly_one_process_refreshes(self, redis):
+ replicas = [provider(redis) for _ in range(5)]
+
+ assert sum(p.should_fetch_flag_definitions() for p in replicas) == 1
+
+ def test_the_refresher_keeps_the_job_across_polls(self, redis):
+ leader, follower = provider(redis), provider(redis)
+ assert leader.should_fetch_flag_definitions() is True
+
+ for _ in range(4):
+ redis.advance(REFRESH)
+ assert follower.should_fetch_flag_definitions() is False
+ assert leader.should_fetch_flag_definitions() is True
+
+ def test_followers_read_what_the_refresher_wrote(self, redis):
+ leader, follower = provider(redis), provider(redis)
+ leader.should_fetch_flag_definitions()
+ leader.on_flag_definitions_received(DEFINITIONS)
+
+ assert follower.should_fetch_flag_definitions() is False
+ assert follower.get_flag_definitions() == DEFINITIONS
+
+ def test_a_dead_refresher_hands_over_once_its_lock_expires(self, redis):
+ leader, standby = provider(redis), provider(redis)
+ leader.should_fetch_flag_definitions()
+ assert standby.should_fetch_flag_definitions() is False
+
+ redis.advance(LOCK_TTL + 1) # the leader stopped renewing
+
+ assert standby.should_fetch_flag_definitions() is True
+
+ def test_a_graceful_shutdown_hands_over_at_once(self, redis):
+ leader, standby = provider(redis), provider(redis)
+ leader.should_fetch_flag_definitions()
+
+ leader.shutdown()
+
+ assert standby.should_fetch_flag_definitions() is True
+
+ def test_a_follower_shutting_down_leaves_the_lock_alone(self, redis):
+ leader, follower = provider(redis), provider(redis)
+ leader.should_fetch_flag_definitions()
+ follower.should_fetch_flag_definitions()
+
+ follower.shutdown()
+
+ assert provider(redis).should_fetch_flag_definitions() is False
+ assert leader.should_fetch_flag_definitions() is True
+
+ def test_a_stale_refresher_shutting_down_leaves_the_new_lock_alone(self, redis):
+ """It paused, its lock expired, another took it — it must not free that."""
+ stalled, replacement = provider(redis), provider(redis)
+ stalled.should_fetch_flag_definitions()
+ redis.advance(LOCK_TTL + 1)
+ assert replacement.should_fetch_flag_definitions() is True
+
+ stalled.shutdown()
+
+ assert provider(redis).should_fetch_flag_definitions() is False
+
+ def test_becoming_the_refresher_says_so(self, redis, logs):
+ provider(redis).should_fetch_flag_definitions()
+
+ assert any("now refreshes the shared PostHog flag" in m for m in logs)
+
+
+class TestFailureModes:
+ def test_an_empty_cache_reads_as_a_miss(self, redis):
+ assert provider(redis).get_flag_definitions() is None
+
+ @pytest.mark.parametrize(
+ "call",
+ [
+ lambda p: p.get_flag_definitions(),
+ lambda p: p.on_flag_definitions_received(DEFINITIONS),
+ lambda p: p.shutdown(),
+ ],
+ ids=["read", "write", "shutdown"],
+ )
+ def test_redis_being_down_never_raises(self, redis, call):
+ p = provider(redis)
+ p.should_fetch_flag_definitions()
+ redis.fail = True
+
+ assert call(p) is None
+
+ def test_redis_being_down_falls_back_to_fetching(self, redis):
+ redis.fail = True
+
+ assert provider(redis).should_fetch_flag_definitions() is True
+
+ def test_redis_being_down_is_logged_once(self, redis, logs):
+ redis.fail = True
+ p = provider(redis)
+
+ for _ in range(10):
+ p.should_fetch_flag_definitions()
+
+ assert sum("Redis unavailable" in m for m in logs) == 1
+
+ def test_definitions_past_the_stale_window_are_served_with_a_warning(
+ self, redis, logs
+ ):
+ store_definitions(redis, age_seconds=REFRESH * cache._STALE_AFTER_POLLS + 60)
+
+ assert provider(redis).get_flag_definitions() == DEFINITIONS
+ assert any("are " in m and "s old" in m for m in logs)
+
+ def test_fresh_definitions_come_with_no_warning(self, redis, logs):
+ store_definitions(redis, age_seconds=1)
+
+ assert provider(redis).get_flag_definitions() == DEFINITIONS
+ assert not any("s old" in m for m in logs)
+
+ @pytest.mark.parametrize(
+ "content", ["not json", json.dumps({"definitions": DEFINITIONS})]
+ )
+ def test_unreadable_contents_are_discarded(self, redis, content):
+ redis.set(cache._DATA_KEY, content)
+
+ assert provider(redis).get_flag_definitions() is None
+
+ def test_the_shared_copy_expires(self, redis):
+ p = provider(redis, ttl=600)
+ p.on_flag_definitions_received(DEFINITIONS)
+
+ redis.advance(601)
+
+ assert p.get_flag_definitions() is None
+
+
+class TestFailFastRedis:
+ """An optional cache must not hold the SDK's poller thread on a dead Redis."""
+
+ def test_it_connects_once_with_short_timeouts(self, mocker):
+ connect = mocker.patch.object(
+ cache, "connect_once", side_effect=ConnectionError("redis is down")
+ )
+ p = cache.RedisFlagDefinitionCache(refresh_interval=REFRESH, ttl=600)
+
+ assert p.should_fetch_flag_definitions() is True
+ connect.assert_called_once_with(timeout=cache._REDIS_TIMEOUT_SECONDS)
+
+ def test_connect_once_gives_up_at_once(self, monkeypatch):
+ monkeypatch.setattr(redis_client, "HOST", "127.0.0.1")
+ monkeypatch.setattr(redis_client, "PORT", 1)
+ started = time.monotonic()
+
+ with pytest.raises(Exception):
+ redis_client.connect_once(timeout=0.5)
+ assert time.monotonic() - started < 5
+
+ def test_a_failure_drops_the_client_for_a_fresh_one(self, redis):
+ connects = []
+ p = cache.RedisFlagDefinitionCache(
+ refresh_interval=REFRESH,
+ ttl=600,
+ redis_factory=lambda: connects.append(1) or redis,
+ )
+ p.should_fetch_flag_definitions()
+ p.should_fetch_flag_definitions()
+ assert len(connects) == 1
+
+ redis.fail = True
+ p.should_fetch_flag_definitions()
+ redis.fail = False
+ p.should_fetch_flag_definitions()
+
+ assert len(connects) == 2
+
+ def test_shutdown_never_connects(self, redis):
+ connects = []
+ p = cache.RedisFlagDefinitionCache(
+ refresh_interval=REFRESH,
+ ttl=600,
+ redis_factory=lambda: connects.append(1) or redis,
+ )
+ p.should_fetch_flag_definitions()
+ redis.fail = True
+ p.get_flag_definitions()
+
+ p.shutdown()
+
+ assert len(connects) == 1
+
+
+class TestMemoryCache:
+ def test_it_refreshes_once_then_serves_itself(self):
+ p = cache.MemoryFlagDefinitionCache(refresh_interval=REFRESH)
+
+ assert p.should_fetch_flag_definitions() is True
+ p.on_flag_definitions_received(DEFINITIONS)
+
+ assert p.should_fetch_flag_definitions() is False
+ assert p.get_flag_definitions() == DEFINITIONS
+
+
+class TestSelection:
+ @pytest.mark.parametrize(
+ "backend, expected",
+ [
+ (FlagDefinitionCacheBackend.REDIS, cache.RedisFlagDefinitionCache),
+ (FlagDefinitionCacheBackend.MEMORY, cache.MemoryFlagDefinitionCache),
+ (FlagDefinitionCacheBackend.NONE, type(None)),
+ ],
+ )
+ def test_the_setting_picks_the_provider(self, mocker, backend, expected):
+ mocker.patch.object(
+ cache.settings.config, "posthog_flag_definition_cache", backend
+ )
+
+ assert isinstance(cache.get_flag_definition_cache(), expected)
+
+ @pytest.mark.parametrize("field", ["refresh_seconds", "cache_ttl_seconds"])
+ @pytest.mark.parametrize("value", [0, -1])
+ def test_a_non_positive_duration_is_refused_at_startup(
+ self, monkeypatch, field, value
+ ):
+ """0 clamped the poll to one second and made Redis reject every write."""
+ # Init kwargs rank below the environment and .env, which carry these keys.
+ monkeypatch.setenv(f"POSTHOG_FLAG_DEFINITION_{field.upper()}", str(value))
+ with pytest.raises(ValidationError):
+ Config(_env_file=None)
+
+ def test_redis_is_the_default(self):
+ assert (
+ cache.settings.config.posthog_flag_definition_cache
+ is FlagDefinitionCacheBackend.REDIS
+ )
+
+
+class TestClientWiring:
+ """What the PostHog client is actually built with."""
+
+ @pytest.fixture(autouse=True)
+ def fresh_client(self, mocker):
+ mocker.patch.object(ph, "_client", None)
+ mocker.patch.object(ph, "_init_attempted", False)
+ mocker.patch.object(ph.settings.secrets, "posthog_api_key", "phc_test")
+ mocker.patch.object(
+ ph.settings.secrets, "posthog_personal_api_key", "phx_personal"
+ )
+
+ def test_the_client_shares_its_definitions_through_the_cache(self, mocker):
+ posthog = mocker.patch.object(ph, "Posthog")
+
+ ph.get_flag_client()
+
+ _, kwargs = posthog.call_args
+ assert isinstance(
+ kwargs["flag_definition_cache_provider"], cache.RedisFlagDefinitionCache
+ )
+ assert kwargs["poll_interval"] == REFRESH
+
+ def test_no_personal_key_means_no_poller_to_share(self, mocker):
+ mocker.patch.object(ph.settings.secrets, "posthog_personal_api_key", "")
+ posthog = mocker.patch.object(ph, "Posthog")
+
+ ph.get_flag_client()
+
+ _, kwargs = posthog.call_args
+ assert kwargs["flag_definition_cache_provider"] is None
+ assert kwargs["enable_local_evaluation"] is False
+
+
+class TestDefaultBackendIsUntouched:
+ """`launchdarkly` is the default, and it reaches none of this."""
+
+ @pytest.fixture
+ def ld_client(self, mocker):
+ client = mocker.Mock(spec=LDClient)
+ mocker.patch("backend.util.feature_flag.ldclient.get", return_value=client)
+ client.is_initialized.return_value = True
+ return client
+
+ @pytest.fixture
+ def user_context(self, mocker):
+ context = Context.builder("u-1").kind("user").anonymous(False).build()
+ return mocker.patch(
+ "backend.util.feature_flag._fetch_user_context_status",
+ return_value=(context, True),
+ )
+
+ def test_the_default_flag_backend_is_launchdarkly(self):
+ assert (
+ ff.settings.config.feature_flag_backend is FeatureFlagBackend.LAUNCHDARKLY
+ )
+
+ @pytest.mark.asyncio
+ async def test_the_default_builds_no_definition_cache(
+ self, mocker, ld_client, user_context
+ ):
+ build = mocker.patch.object(ph, "get_flag_definition_cache")
+ ld_client.variation.return_value = True
+
+ assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)
+ build.assert_not_called()
+
+
+class TestAgainstTheRealSDK:
+ """The SDK's own poll, driven through the provider."""
+
+ @pytest.fixture
+ def fetches(self, mocker):
+ """Every definitions fetch the SDK makes, with one canned answer."""
+ return mocker.patch(
+ "posthog.client.get",
+ return_value=GetResponse(data=DEFINITIONS, etag="e1"),
+ )
+
+ def client(self, provider_impl) -> Posthog:
+ return Posthog(
+ "phc_test",
+ personal_api_key="phx_personal",
+ enable_local_evaluation=True,
+ sync_mode=True,
+ poll_interval=REFRESH,
+ flag_definition_cache_provider=provider_impl,
+ )
+
+ def test_a_follower_evaluates_without_fetching_definitions(self, redis, fetches):
+ provider(redis).should_fetch_flag_definitions() # someone else leads
+ store_definitions(redis)
+ follower = self.client(provider(redis))
+
+ follower._load_feature_flags() # what the SDK's poller thread runs
+
+ fetches.assert_not_called()
+ snapshot = follower.evaluate_flags(
+ "u-1", flag_keys=["shared-flag"], only_evaluate_locally=True
+ )
+ assert snapshot.get_flag("shared-flag") is True
+
+ def test_the_refresher_fetches_and_shares_what_it_got(self, redis, fetches):
+ leader = self.client(provider(redis))
+
+ leader._load_feature_flags()
+
+ fetches.assert_called_once()
+ shared = provider(redis).get_flag_definitions()
+ assert shared is not None
+ assert shared["flags"] == DEFINITIONS["flags"]
+
+ def test_unchanged_definitions_keep_the_shared_copy_alive(self, redis, fetches):
+ unchanged = GetResponse(data=None, etag="e1", not_modified=True)
+ fetches.side_effect = [
+ GetResponse(data=DEFINITIONS, etag="e1"),
+ unchanged,
+ unchanged,
+ ]
+ leader = self.client(provider(redis, ttl=LOCK_TTL))
+
+ for _ in range(3):
+ leader._load_feature_flags()
+ redis.advance(REFRESH + 1)
+
+ assert fetches.call_count == 3
+ shared = provider(redis).get_flag_definitions()
+ assert shared is not None
+ assert shared["flags"] == DEFINITIONS["flags"]
+
+ def test_an_empty_cache_falls_back_to_one_direct_fetch(self, redis, fetches):
+ provider(redis).should_fetch_flag_definitions() # someone else leads
+ follower = self.client(provider(redis)) # and has written nothing yet
+
+ follower._load_feature_flags()
+
+ fetches.assert_called_once()
+
+ def test_a_failed_fetch_keeps_the_previous_definitions(self, redis, fetches):
+ leader = self.client(provider(redis))
+ leader._load_feature_flags()
+ fetches.side_effect = ConnectionError("PostHog is unreachable")
+
+ leader._load_feature_flags()
+
+ snapshot = leader.evaluate_flags(
+ "u-1", flag_keys=["shared-flag"], only_evaluate_locally=True
+ )
+ assert snapshot.get_flag("shared-flag") is True
+
+ def test_redis_being_down_leaves_every_process_fetching_for_itself(
+ self, redis, fetches
+ ):
+ redis.fail = True
+ follower = self.client(provider(redis))
+
+ follower._load_feature_flags()
+
+ fetches.assert_called_once()
+
+
+ not is_tcp_port_reachable(redis_client.HOST, redis_client.PORT),
+ reason="no local Redis reachable; the lock scripts need one to run",
+)
+class TestTheLockScriptsOnRealRedis:
+ """FakeRedis stands in for Redis's semantics; only Redis runs the Lua."""
+
+ @pytest.fixture
+ def key(self):
+ key = f"test:flag_definitions:refresher:{uuid.uuid4().hex}"
+ yield key
+ redis_client.get_redis().delete(key)
+
+ def test_renewing_extends_only_our_own_lock(self, key):
+ redis = redis_client.get_redis()
+ redis.set(key, "holder", px=5_000)
+
+ assert redis.eval(cache._RENEW_LOCK, 1, key, "someone-else", 60_000) == 0
+ assert redis.pttl(key) <= 5_000
+
+ assert redis.eval(cache._RENEW_LOCK, 1, key, "holder", 60_000) == 1
+ assert redis.pttl(key) > 5_000
+
+ def test_releasing_frees_only_our_own_lock(self, key):
+ redis = redis_client.get_redis()
+ redis.set(key, "holder", px=60_000)
+
+ assert redis.eval(cache._RELEASE_LOCK, 1, key, "someone-else") == 0
+ assert redis.get(key) == "holder"
+
+ assert redis.eval(cache._RELEASE_LOCK, 1, key, "holder") == 1
+ assert redis.get(key) is None
+
+
+class TestMetrics:
+ def test_a_cache_read_is_counted(self, redis):
+ from prometheus_client import REGISTRY
+
+ name = "autogpt_posthog_flag_definition_cache_events_total"
+ before = REGISTRY.get_sample_value(name, {"outcome": "cached"}) or 0
+ store_definitions(redis)
+
+ provider(redis).get_flag_definitions()
+
+ assert REGISTRY.get_sample_value(name, {"outcome": "cached"}) == before + 1
diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog.py b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
index b6ee306548..24625e21b4 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/posthog.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
@@ -15,6 +15,8 @@ from posthog import Posthog
from backend.util.settings import Settings
+from .definition_cache import get_flag_definition_cache, refresh_interval_seconds
+
logger = logging.getLogger(__name__)
settings = Settings()
@@ -143,15 +145,21 @@ def _build_client() -> Posthog | None:
return None
personal_api_key = settings.secrets.posthog_personal_api_key
+ # One shared refresher, so the definitions bill stops scaling with replica
+ # count. Without a provider the SDK polls PostHog once per process.
+ definition_cache = get_flag_definition_cache() if personal_api_key else None
_client = Posthog(
settings.secrets.posthog_api_key,
host=settings.secrets.posthog_host,
personal_api_key=personal_api_key or None,
enable_local_evaluation=bool(personal_api_key),
+ poll_interval=refresh_interval_seconds(),
+ flag_definition_cache_provider=definition_cache,
)
_init_attempted = True
logger.info(
"PostHog feature flag client initialized "
- f"(local evaluation: {'on' if personal_api_key else 'off'})"
+ f"(local evaluation: {'on' if personal_api_key else 'off'}, "
+ f"definition cache: {type(definition_cache).__name__ if definition_cache else 'off'})"
)
return _client
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index 6db5487ff3..e38eed34ee 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -63,6 +63,19 @@ class FeatureFlagBackend(str, Enum):
DUAL = "dual"
+class FlagDefinitionCacheBackend(str, Enum):
+ """Where PostHog flag definitions are shared between processes.
+
+ PostHog bills one definitions fetch as ten flag requests, so a poller in
+ every process makes the bill scale with replica count. ``REDIS`` elects one
+ refresher and serves every other process from its copy.
+ """
+
+ REDIS = "redis"
+ MEMORY = "memory"
+ NONE = "none"
+
+
class UpdateTrackingModel(BaseModel, Generic[T]):
_updated_fields: Set[str] = PrivateAttr(default_factory=set)
@@ -769,6 +782,44 @@ class Config(UpdateTrackingModel["Config"], BaseSettings):
)
return FeatureFlagBackend.LAUNCHDARKLY
+ posthog_flag_definition_cache: FlagDefinitionCacheBackend = Field(
+ default=FlagDefinitionCacheBackend.REDIS,
+ description="Where PostHog flag definitions are shared: redis "
+ "(default; one elected refresher, every other process reads its copy), "
+ "memory (process-local), or none (every process polls PostHog itself). "
+ "Only read when PostHog answers flag reads.",
+ )
+
+ @field_validator("posthog_flag_definition_cache", mode="before")
+ @classmethod
+ def _default_unknown_definition_cache(cls, v):
+ """Same reasoning as ``_default_unknown_flag_backend`` above."""
+ if not isinstance(v, str):
+ return v
+ try:
+ return FlagDefinitionCacheBackend(v.strip().lower())
+ except ValueError:
+ logger.warning(
+ f"Unknown POSTHOG_FLAG_DEFINITION_CACHE {v!r}, "
+ f"falling back to {FlagDefinitionCacheBackend.REDIS.value}"
+ )
+ return FlagDefinitionCacheBackend.REDIS
+
+ posthog_flag_definition_refresh_seconds: int = Field(
+ default=30,
+ ge=1,
+ description="How often the elected refresher fetches PostHog flag "
+ "definitions, and how often every other process re-reads the shared copy.",
+ )
+
+ posthog_flag_definition_cache_ttl_seconds: int = Field(
+ default=600,
+ ge=1,
+ description="How long a shared copy of the PostHog flag definitions stays "
+ "readable. Past it the cache is empty and the next process to poll fetches "
+ "from PostHog directly.",
+ )
+
behave_as: BehaveAs = Field(
default=BehaveAs.LOCAL,
description="What environment to behave as: local or cloud",
diff --git a/docs/platform/feature-flags.md b/docs/platform/feature-flags.md
new file mode 100644
index 0000000000..3d48cf3d05
--- /dev/null
+++ b/docs/platform/feature-flags.md
@@ -0,0 +1,92 @@
+# Feature flags
+
+Which vendor answers a flag read is set per deployment by `FEATURE_FLAG_BACKEND`:
+`launchdarkly` (the default), `posthog`, or `dual` — evaluate both, serve
+LaunchDarkly's answer, and log every disagreement. Call sites are unaffected by
+the choice; they read through `evaluate_feature_flag()` on the backend and
+`useGetFlag` / `useFlagStatus` on the frontend.
+
+## Sharing PostHog's flag definitions
+
+PostHog evaluates flags locally, in the process, from a set of flag definitions
+it polls for every 30 seconds. PostHog
+[bills one definitions fetch as ten flag requests](https://posthog.com/docs/feature-flags/local-evaluation),
+so a poller in every process makes that bill scale with replica count: at two
+polls a minute, each process costs 28,800 flag requests a day whether or not it
+reads a single flag.
+
+So one process is elected to refresh the definitions and write them to Redis,
+and every other process evaluates from that shared copy. The fleet then costs
+one process's worth of polling — 28,800 flag requests a day in total, instead of
+28,800 per replica.
+
+This is PostHog's own `FlagDefinitionCacheProvider` interface; the SDK calls it
+from its polling thread, never from a flag read, so no request can block on
+Redis.
+
+### Settings
+
+| Setting | Default | What it does |
+|---|---|---|
+| `POSTHOG_FLAG_DEFINITION_CACHE` | `redis` | `redis` shares definitions through Redis; `memory` keeps them in the process; `none` restores one poller per process. |
+| `POSTHOG_FLAG_DEFINITION_REFRESH_SECONDS` | `30` | How often the refresher fetches, and how often everyone else re-reads the shared copy. |
+| `POSTHOG_FLAG_DEFINITION_CACHE_TTL_SECONDS` | `600` | How long the shared copy stays readable after the last refresh. |
+
+None of them is read unless `FEATURE_FLAG_BACKEND` is `posthog` or `dual` **and**
+`POSTHOG_PERSONAL_API_KEY` is set — without that key the SDK evaluates over the
+wire and has no definitions to share.
+
+**Per environment.** Leave the defaults on `dev` and `prod`: both run several
+replicas against a shared Redis, which is what the cache is for. Local
+development and CI run `FEATURE_FLAG_BACKEND=launchdarkly`, so none of this is
+reached; set `POSTHOG_FLAG_DEFINITION_CACHE=memory` if you want to exercise
+PostHog locally without Redis.
+
+### What the refresher is
+
+Every process tries `SET <key> <id> NX PX` on one Redis key each time it polls.
+The one that gets it fetches the definitions from PostHog and writes them to
+Redis; the others read that copy. The winner renews its own lock on every poll,
+so it keeps the job. The lock expires two poll intervals after its last renewal,
+and the next process to poll takes it, so a refresher that dies is replaced
+within 90 seconds at the defaults. One shutting down cleanly releases the lock
+immediately, and is replaced on the next poll.
+
+### Telling it is working
+
+One process logs, once, when it takes the job:
+
+```
+This process now refreshes the shared PostHog flag definitions (<host>:<pid>:<id>)
+```
+
+and the others log `Another process now refreshes the shared PostHog flag
+definitions` when they lose it.
+
+The counter `autogpt_posthog_flag_definition_cache_events_total` on `/metrics`
+carries the same story, by `outcome`:
+
+| outcome | meaning |
+|---|---|
+| `refresher` / `stored` | this process holds the lock and wrote its definitions to the shared copy, which it does every poll whether or not PostHog reported a change |
+| `follower` / `cached` | this process read the shared copy |
+| `stale` | the shared copy has not been rewritten for five poll intervals, so no refresher is writing it; it is served anyway |
+| `empty` | nothing in the cache; a process with no definitions yet fetches from PostHog directly, and one that has some keeps them |
+| `error` | Redis was unreachable |
+
+Healthy, with N replicas: `stored` increments once per poll on exactly one
+process, `cached` once per poll on each of the other N−1, and `empty` and
+`error` stay flat.
+
+### Failure modes
+
+| What happens | What the platform does |
+|---|---|
+| Cache empty at boot | The process fetches from PostHog directly, once, and the refresher fills the cache. |
+| Redis unreachable | Every process polls PostHog for itself, as it would without the cache, and logs once per five minutes. Flag reads are unaffected. |
+| Shared copy goes stale | It is served anyway, with a warning, while the refresher catches up. Stale definitions beat none. |
+| The refresher dies | Its lock expires and another process takes the job on its next poll. |
+| A definitions fetch fails | The definitions already in memory stay in use, and the refresher keeps writing that copy to the cache. |
+
+Nothing here can make a flag read wait on Redis or on PostHog: flag evaluation
+only ever reads the definitions already in the process.