Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(backend): stop asking PostHog over the network about flags it doesn't have (#15765)

Nicholas Tindleabcac991599f
diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog.py b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
index a95b2d443d..6c0541fe67 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/posthog.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
@@ -9,6 +9,8 @@ import functools
import json
import logging
import threading
+import time
+import weakref
from concurrent.futures import ThreadPoolExecutor
from typing import Any
@@ -44,6 +46,18 @@ _client_lock = threading.Lock()
_init_attempted = False
_shut_down = False
+# The definitions a client polls leave out some flags PostHog still answers
+# over /flags (one in its newer config format, for instance), so a read of a
+# flag they lack asks /flags. Once /flags has had no answer for one either (a
+# flag nobody created), reads of it stop asking for this long: each ask is a
+# network round trip on the caller's path.
+UNANSWERED_RECHECK_SECONDS = 300
+# Per client: each flag its definitions lack that /flags had no answer for,
+# with when that was learned.
+_unanswered: weakref.WeakKeyDictionary[Posthog, dict[str, float]] = (
+ weakref.WeakKeyDictionary()
+)
+
def is_configured() -> bool:
"""Whether PostHog has the project key a flag read needs."""
@@ -73,6 +87,8 @@ async def evaluate_flag(
if client is None:
logger.debug(f"PostHog not configured, using default={default} for {flag_key}")
return default, False
+ if _known_unanswered(client, flag_key):
+ return default, False
try:
# The read does network I/O whenever the local-evaluation poller has
@@ -95,6 +111,7 @@ async def evaluate_flag(
if result is None:
logger.debug(f"PostHog returned no answer for {flag_key}, using {default}")
+ _unanswered.setdefault(client, {})[flag_key] = time.monotonic()
return default, False
# A conclusive "off" is the answer. Serving a payload here instead would
@@ -247,3 +264,13 @@ def _decoded(payload: Any, value: Any) -> Any:
def _loaded_empty() -> bool:
last = _definition_cache.definitions if _definition_cache else None
return last is not None and not last.get("flags")
+
+
+def _known_unanswered(client: Posthog, flag_key: str) -> bool:
+ """Whether /flags recently had no answer for a flag the definitions lack."""
+ learned = _unanswered.get(client, {}).get(flag_key)
+ if learned is None or time.monotonic() - learned > UNANSWERED_RECHECK_SECONDS:
+ return False
+ # One created since is read from its definition from the next poll on.
+ defined = defined_flag_keys()
+ return defined is not None and flag_key not in defined
diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
index b536542dfb..2bb3f2ff63 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
@@ -7,7 +7,7 @@ from concurrent.futures import ThreadPoolExecutor
import pytest
from posthog import Posthog
-from posthog.types import FeatureFlagResult
+from posthog.types import FeatureFlag, FeatureFlagResult
import backend.util.feature_flag.posthog as ph
@@ -340,6 +340,110 @@ class TestRawRead:
loop.close()
+class TestAFlagTheDefinitionsLeaveOut:
+ """Only PostHog's /flags can answer one, at a network call per read."""
+
+ @pytest.fixture
+ def client(self, mocker):
+ client = Posthog(
+ "phc-test", secret_key="phx-test", enable_local_evaluation=False
+ )
+ client.feature_flags = [_rolled_out("hire-experts", 100)]
+ mocker.patch.object(ph, "get_flag_client", return_value=client)
+ yield client
+ client.shutdown()
+
+ @pytest.mark.asyncio
+ async def test_one_posthog_has_no_answer_for_is_asked_about_once(
+ self, mocker, client
+ ):
+ remote = mocker.patch.object(
+ client, "_get_flags_decision", return_value={"flags": {}}
+ )
+
+ reads = [
+ await ph.evaluate_flag("copilot-sdk", "u-1", default=True) for _ in range(3)
+ ]
+
+ assert reads == [(True, False)] * 3
+ assert remote.call_count == 1
+
+ @pytest.mark.asyncio
+ async def test_it_is_asked_about_again_once_that_answer_is_stale(
+ self, mocker, client
+ ):
+ mocker.patch.object(ph, "UNANSWERED_RECHECK_SECONDS", -1)
+ remote = mocker.patch.object(
+ client, "_get_flags_decision", return_value={"flags": {}}
+ )
+
+ for _ in range(2):
+ await ph.evaluate_flag("copilot-sdk", "u-1", default=True)
+
+ assert remote.call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_one_posthog_answers_is_asked_about_every_time(self, mocker, client):
+ """As a flag in PostHog's newer config format is: the definitions SDKs
+ poll leave it out, and only /flags evaluates it."""
+ remote = mocker.patch.object(
+ client,
+ "_get_flags_decision",
+ return_value={
+ "flags": {
+ "copilot-skills": FeatureFlag.from_value_and_payload(
+ "copilot-skills", False, None
+ )
+ }
+ },
+ )
+
+ reads = [
+ await ph.evaluate_flag("copilot-skills", "u-1", default=True)
+ for _ in range(2)
+ ]
+
+ assert reads == [(False, True)] * 2
+ assert remote.call_count == 2
+
+ @pytest.mark.asyncio
+ async def test_a_flag_defined_since_is_read_from_its_definition(
+ self, mocker, client
+ ):
+ """Creating a default-on flag switched off turns its feature off at the
+ next definitions poll, not when PostHog's earlier answer goes stale."""
+ remote = mocker.patch.object(
+ client, "_get_flags_decision", return_value={"flags": {}}
+ )
+ await ph.evaluate_flag("copilot-skills", "u-1", default=True)
+
+ client.feature_flags = [
+ _rolled_out("hire-experts", 100),
+ {**_rolled_out("copilot-skills", 100), "id": 2, "active": False},
+ ]
+
+ assert await ph.evaluate_flag("copilot-skills", "u-1", default=True) == (
+ False,
+ True,
+ )
+ assert remote.call_count == 1
+
+ @pytest.mark.asyncio
+ async def test_every_read_asks_while_the_definitions_are_unknown(
+ self, mocker, client
+ ):
+ mocker.patch.object(ph, "_definition_cache", None)
+ client.feature_flags = []
+ remote = mocker.patch.object(
+ client, "_get_flags_decision", return_value={"flags": {}}
+ )
+
+ for _ in range(2):
+ await ph.evaluate_flag("copilot-sdk", "u-1", default=True)
+
+ assert remote.call_count == 2
+
+
def _rolled_out(key: str, percentage: int, payload: str | None = None) -> dict:
filters: dict = {"groups": [{"properties": [], "rollout_percentage": percentage}]}
if payload is not None:
diff --git a/docs/engineering/feature-flags.md b/docs/engineering/feature-flags.md
index 71282bae9e..9eacdafe0b 100644
--- a/docs/engineering/feature-flags.md
+++ b/docs/engineering/feature-flags.md
@@ -17,6 +17,14 @@ project's Feature Flags Secure API Key in `POSTHOG_SECRET_KEY`. Where local
evaluation can't decide a read, the SDK asks PostHog's `/flags` endpoint
instead, unless the read is local-only, as a signed-out visitor's own read is.
+A flag the code reads but the project doesn't have is served its code default.
+The backend still asks `/flags` about it once, because PostHog answers some
+flags there that it leaves out of the definitions (one in its newer config
+format, for instance). Once `/flags` has had no answer either, the backend
+serves the default without asking again for five minutes, so the read costs no
+network round trip. Creating the flag takes effect at the next definitions
+poll, or within those five minutes for a flag the definitions leave out.
+
| | Dev | Prod |
|---|---|---|
| PostHog project | [281733](https://eu.posthog.com/project/281733/feature_flags) | [117338](https://eu.posthog.com/project/117338/feature_flags) |