Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

feat(backend): read PostHog flags from one project while dev reports to its own (#15483)

Krzysztof Czerwinskib637f70614c9
diff --git a/autogpt_platform/backend/.env.default b/autogpt_platform/backend/.env.default
index 4eacd7b4d4..3e617341ad 100644
--- a/autogpt_platform/backend/.env.default
+++ b/autogpt_platform/backend/.env.default
@@ -390,9 +390,15 @@ ZEROBOUNCE_API_KEY=
# Get API key from https://posthog.com - Project Settings > Project API Key
POSTHOG_API_KEY=
POSTHOG_HOST=https://eu.i.posthog.com
-# Feature Flags Secure API Key, only for local feature-flag evaluation (project
-# settings > Feature Flags). A personal API key also works, but PostHog is
-# deprecating that use. Without it flag reads go over the wire.
+# Project API key of the project feature flags are read from, when that is not
+# the analytics project above. Dev sets prod's key here (one set of flags) while
+# its events stay in the dev project. Blank reads flags from POSTHOG_API_KEY's.
+POSTHOG_FLAGS_API_KEY=
+# Feature Flags Secure API Key of the project flags are read from, only for
+# local feature-flag evaluation (project settings > Feature Flags). A personal
+# API key also works, but PostHog is deprecating that use. Without it flag reads
+# go over the wire, or serve their defaults when POSTHOG_FLAGS_API_KEY names
+# another project.
POSTHOG_SECRET_KEY=
# Where flag definitions are shared between processes: redis (default; one
# elected refresher, every other process reads its copy), memory (process-local),
diff --git a/autogpt_platform/backend/backend/util/feature_flag/__init__.py b/autogpt_platform/backend/backend/util/feature_flag/__init__.py
index 583dffaf1b..30daf4851f 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/__init__.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/__init__.py
@@ -668,6 +668,19 @@ async def _evaluate_launchdarkly(
def _person_properties(context: Context) -> dict[str, Any]:
+ """PostHog person properties for one read: the user's, plus ``environment``.
+
+ ``environment`` rides on every read, anonymous ones included: dev reads
+ prod's flag definitions, so a rollout to dev alone is a release condition
+ on it. It is the app env the analytics events also carry.
+ """
+ return {
+ **_context_properties(context),
+ "environment": settings.config.app_env.value,
+ }
+
+
+def _context_properties(context: Context) -> dict[str, Any]:
"""PostHog person properties from the shared user context.
Reads the LaunchDarkly context rather than the auth row so both vendors
diff --git a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py b/autogpt_platform/backend/backend/util/feature_flag/backends_test.py
index 5347b3b1ea..76526abb02 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/backends_test.py
@@ -259,6 +259,7 @@ class TestPostHogBackend:
"email": "[email protected]",
"email_domain": "agpt.co",
"created_at": "2026-05-07T12:00:00+00:00",
+ "environment": ff.settings.config.app_env.value,
}
@pytest.mark.asyncio
@@ -291,8 +292,9 @@ class TestPostHogBackend:
assert result == expected
@pytest.mark.asyncio
- async def test_an_anonymous_context_carries_no_person_properties(self, mocker):
- """The `"system"`-keyed config flags have no user to describe."""
+ async def test_an_anonymous_context_carries_only_the_environment(self, mocker):
+ """The `"system"`-keyed config flags have no user to describe, but can
+ still differ between dev and prod."""
use_backend(mocker, FeatureFlagBackend.POSTHOG)
client = stub_posthog(mocker, value=True, payload={"TIER": 1.5})
@@ -300,7 +302,33 @@ class TestPostHogBackend:
args, kwargs = client.evaluate_flags.call_args
assert args[0] == "system"
- assert kwargs["person_properties"] is None
+ assert kwargs["person_properties"] == {
+ "environment": ff.settings.config.app_env.value
+ }
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "app_env,expected",
+ [
+ (AppEnvironment.DEVELOPMENT, (True, True)),
+ (AppEnvironment.PRODUCTION, (False, True)),
+ ],
+ )
+ async def test_a_dev_only_rollout_evaluates_in_process(
+ self, mocker, user_context, app_env, expected
+ ):
+ """Dev reads prod's definitions; a release condition on ``environment``
+ is how a flag goes out to dev alone."""
+ use_backend(mocker, FeatureFlagBackend.POSTHOG)
+ mocker.patch.object(ff.settings.config, "app_env", app_env)
+ posthog_client = _posthog_rolled_out_to_dev_only()
+ mocker.patch.object(ph, "get_flag_client", return_value=posthog_client)
+ try:
+ result = await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1")
+ finally:
+ posthog_client.shutdown()
+
+ assert result == expected
def test_lifecycle_starts_only_posthog(self, mocker):
use_backend(mocker, FeatureFlagBackend.POSTHOG)
@@ -1065,6 +1093,35 @@ def _posthog_with_an_employee_cohort() -> Posthog:
return client
+def _posthog_rolled_out_to_dev_only() -> Posthog:
+ """A flag whose one release condition is ``environment = dev``."""
+ client = Posthog("phc-test", secret_key="phx-test", enable_local_evaluation=False)
+ client.feature_flags = [
+ {
+ "id": 1,
+ "key": Flag.HIRE_EXPERTS.value,
+ "active": True,
+ "filters": {
+ "groups": [
+ {
+ "properties": [
+ {
+ "key": "environment",
+ "type": "person",
+ "operator": "exact",
+ "value": ["dev"],
+ }
+ ],
+ "rollout_percentage": 100,
+ }
+ ]
+ },
+ }
+ ]
+ client._get_flags_decision = _no_remote_flags
+ return client
+
+
def _no_remote_flags(*args, **kwargs):
raise ConnectionError("remote /flags is unavailable in tests")
diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog.py b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
index e044a898fc..413d2c19ab 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/posthog.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog.py
@@ -38,7 +38,22 @@ _shut_down = False
def is_configured() -> bool:
"""Whether PostHog has the project key a flag read needs."""
- return bool(settings.secrets.posthog_api_key)
+ return bool(_flag_project_key())
+
+
+def reads_another_project() -> bool:
+ """Whether flags come from a PostHog project other than the analytics one.
+
+ Dev reads prod's flag definitions, so nothing a read does may create data
+ there: no ``$feature_flag_called`` events, and no remote ``/flags`` call
+ carrying a user's properties when local evaluation cannot answer.
+ """
+ flags_key = settings.secrets.posthog_flags_api_key
+ return bool(flags_key) and flags_key != settings.secrets.posthog_api_key
+
+
+def _flag_project_key() -> str:
+ return settings.secrets.posthog_flags_api_key or settings.secrets.posthog_api_key
async def evaluate_flag(
@@ -72,6 +87,7 @@ async def evaluate_flag(
distinct_id,
person_properties=person_properties or None,
flag_keys=[flag_key],
+ only_evaluate_locally=reads_another_project(),
),
)
except Exception as e:
@@ -159,21 +175,32 @@ def _build_client() -> Posthog | None:
return None
secret_key = settings.secrets.posthog_secret_key
+ separate_project = reads_another_project()
+ if separate_project and not secret_key:
+ logger.warning(
+ "POSTHOG_FLAGS_API_KEY names another PostHog project but "
+ "POSTHOG_SECRET_KEY is unset: with no local definitions and no remote "
+ "fallback, every flag read serves its default"
+ )
# One shared refresher, so the definitions bill stops scaling with replica
# count. Without a provider the SDK polls PostHog once per process.
definition_cache = get_flag_definition_cache() if secret_key else None
_client = Posthog(
- settings.secrets.posthog_api_key,
+ _flag_project_key(),
host=settings.secrets.posthog_host,
secret_key=secret_key or None,
enable_local_evaluation=bool(secret_key),
poll_interval=refresh_interval_seconds(),
flag_definition_cache_provider=definition_cache,
+ # Its only events are $feature_flag_called, which would land in the
+ # flag project; this deployment's analytics belong in its own.
+ send=not separate_project,
)
_init_attempted = True
logger.info(
"PostHog feature flag client initialized "
- f"(local evaluation: {'on' if secret_key else 'off'}, "
+ f"(flag project: {'separate' if separate_project else 'analytics'}, "
+ f"local evaluation: {'on' if secret_key else 'off'}, "
f"definition cache: {type(definition_cache).__name__ if definition_cache else 'off'})"
)
return _client
diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
index 051eece9bd..aed68dd93f 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py
@@ -6,9 +6,14 @@ import time
from concurrent.futures import ThreadPoolExecutor
import pytest
+from posthog import Posthog
import backend.util.feature_flag.posthog as ph
+# Taken at import, before any fixture can stub it: the test below needs the
+# SDK's own capture path to show that send=False keeps events off the wire.
+_REAL_CAPTURE = Posthog.capture
+
@pytest.fixture(autouse=True)
def fresh_client(mocker):
@@ -18,9 +23,10 @@ def fresh_client(mocker):
mocker.patch.object(ph, "_shut_down", False)
-def configure(mocker, *, api_key="phc_test", secret_key=""):
+def configure(mocker, *, api_key="phc_test", secret_key="", flags_key=""):
mocker.patch.object(ph.settings.secrets, "posthog_api_key", api_key)
mocker.patch.object(ph.settings.secrets, "posthog_secret_key", secret_key)
+ mocker.patch.object(ph.settings.secrets, "posthog_flags_api_key", flags_key)
def snapshot(mocker, *, value, payload=None):
@@ -150,6 +156,125 @@ class TestClientConstruction:
assert ph.get_flag_client() is not None
+class TestTheFlagProject:
+ """Dev reads prod's flag definitions while its analytics stay in dev."""
+
+ def test_without_a_flags_key_flags_come_from_the_analytics_project(self, mocker):
+ configure(mocker, api_key="phc_analytics")
+ posthog = mocker.patch.object(ph, "Posthog")
+
+ ph.get_flag_client()
+
+ args, kwargs = posthog.call_args
+ assert args[0] == "phc_analytics"
+ assert kwargs["send"] is True
+ assert ph.reads_another_project() is False
+
+ def test_the_same_key_twice_is_one_project(self, mocker):
+ configure(mocker, api_key="phc_prod", flags_key="phc_prod")
+ posthog = mocker.patch.object(ph, "Posthog")
+
+ ph.get_flag_client()
+
+ args, kwargs = posthog.call_args
+ assert args[0] == "phc_prod"
+ assert kwargs["send"] is True
+
+ def test_a_flags_key_reads_that_project_and_sends_it_nothing(self, mocker):
+ configure(
+ mocker, api_key="phc_dev", flags_key="phc_prod", secret_key="phs_prod"
+ )
+ posthog = mocker.patch.object(ph, "Posthog")
+
+ ph.get_flag_client()
+
+ args, kwargs = posthog.call_args
+ assert args[0] == "phc_prod"
+ assert kwargs["secret_key"] == "phs_prod"
+ assert kwargs["send"] is False
+
+ def test_a_flags_key_alone_configures_flag_reads(self, mocker):
+ configure(mocker, api_key="", flags_key="phc_prod")
+
+ assert ph.is_configured() is True
+ assert ph.reads_another_project() is True
+
+ def test_a_separate_project_without_the_secret_key_warns(self, mocker):
+ configure(mocker, api_key="phc_dev", flags_key="phc_prod")
+ mocker.patch.object(ph, "Posthog")
+ warn = mocker.patch.object(ph.logger, "warning")
+
+ ph.get_flag_client()
+
+ assert "POSTHOG_SECRET_KEY" in warn.call_args.args[0]
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "flags_key,only_locally", [("", False), ("phc_test", False), ("phc_prod", True)]
+ )
+ async def test_only_a_separate_project_is_read_locally_only(
+ self, mocker, flags_key, only_locally
+ ):
+ configure(mocker, flags_key=flags_key)
+ client = mocker.Mock()
+ client.evaluate_flags.return_value = snapshot(mocker, value=True)
+ mocker.patch.object(ph, "get_flag_client", return_value=client)
+
+ await ph.evaluate_flag("hire-experts", "u-1")
+
+ _, kwargs = client.evaluate_flags.call_args
+ assert kwargs["only_evaluate_locally"] is only_locally
+
+
+class TestASeparateFlagProjectThroughTheRealSDK:
+ """What the installed posthog-python does with the client built above."""
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "flags_key,asks_remotely", [("", True), ("phc_prod", False)]
+ )
+ async def test_a_flag_missing_locally_goes_remote_only_within_one_project(
+ self, mocker, flags_key, asks_remotely
+ ):
+ """The remote fallback would hand the flag project a user's email."""
+ # The flag-called events are not what this test is about.
+ mocker.patch.object(Posthog, "capture")
+ client, remote = _real_client(mocker, flags_key=flags_key)
+ try:
+ value = await ph.evaluate_flag(
+ "not-in-the-definitions", "u-1", {"email": "[email protected]"}
+ )
+ finally:
+ ph.shutdown_posthog_flags()
+
+ assert value == (None, False)
+ assert remote.called is asks_remotely
+
+ @pytest.mark.asyncio
+ async def test_reads_from_a_separate_project_send_it_no_events(self, mocker):
+ mocker.patch.object(Posthog, "capture", _REAL_CAPTURE)
+ captured = mocker.spy(Posthog, "capture")
+ uploads = [
+ mocker.patch("posthog.consumer.batch_post"),
+ mocker.patch("posthog.consumer._send_v1_batch"),
+ ]
+ client, remote = _real_client(mocker, flags_key="phc_prod")
+ try:
+ assert await ph.evaluate_flag("hire-experts", "u-1") == (True, True)
+ assert await ph.evaluate_flag("not-in-the-definitions", "u-1") == (
+ None,
+ False,
+ )
+ client.flush()
+ finally:
+ ph.shutdown_posthog_flags()
+
+ # The SDK still forms the $feature_flag_called events; none is uploaded.
+ assert captured.called
+ assert not remote.called
+ assert not any(upload.called for upload in uploads)
+
+
class TestRawRead:
@pytest.mark.asyncio
async def test_an_unconfigured_client_returns_the_default(self, mocker):
@@ -282,3 +407,23 @@ class TestRawRead:
assert loop.run_until_complete(scenario()) == "probe"
finally:
loop.close()
+
+
+def _real_client(mocker, *, flags_key: str):
+ """A real flag client holding one rolled-out flag, with /flags recorded."""
+ configure(mocker, api_key="phc_dev", flags_key=flags_key, secret_key="phs_test")
+ mocker.patch.object(ph, "get_flag_definition_cache", return_value=None)
+ client = ph.get_flag_client()
+ assert client is not None
+ client.feature_flags = [
+ {
+ "id": 1,
+ "key": "hire-experts",
+ "active": True,
+ "filters": {"groups": [{"properties": [], "rollout_percentage": 100}]},
+ }
+ ]
+ remote = mocker.patch.object(
+ client, "_get_flags_decision", side_effect=ConnectionError("offline")
+ )
+ return client, remote
diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.py
index c8a7ee8ab2..98988abe9f 100644
--- a/autogpt_platform/backend/backend/util/settings.py
+++ b/autogpt_platform/backend/backend/util/settings.py
@@ -1417,12 +1417,20 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings):
posthog_host: str = Field(
default="https://eu.i.posthog.com", description="PostHog host URL"
)
+ posthog_flags_api_key: str = Field(
+ default="",
+ description="Project API key of the PostHog project feature flags are read "
+ "from, when that is not the analytics project. Dev sets prod's, so both "
+ "share one set of flag definitions while dev's events stay in its own "
+ "project. Blank reads flags from the POSTHOG_API_KEY project.",
+ )
posthog_secret_key: str = Field(
default="",
description="PostHog Feature Flags Secure API Key (project settings > "
- "Feature Flags), for local flag evaluation; without it flag reads fall back "
- "to a remote /flags call. A personal API key also works, but PostHog is "
- "deprecating that use.",
+ "Feature Flags) of the project flags are read from, for local flag "
+ "evaluation; without it flag reads fall back to a remote /flags call, or "
+ "serve their defaults when POSTHOG_FLAGS_API_KEY names another project. "
+ "A personal API key also works, but PostHog is deprecating that use.",
)
# Add more secret fields as needed
diff --git a/autogpt_platform/backend/scripts/feature_flag_sync.py b/autogpt_platform/backend/scripts/feature_flag_sync.py
index 2ef6a8e871..fe5aa4ef9d 100644
--- a/autogpt_platform/backend/scripts/feature_flag_sync.py
+++ b/autogpt_platform/backend/scripts/feature_flag_sync.py
@@ -24,6 +24,8 @@ from pydantic import BaseModel
# `country` is not stored on the person: it is the visitor's ISO country code,
# passed per evaluation by callers that know it (the trial offer, from the
# country token the proxy signs), so it is absent whenever a caller does not.
+# `environment` is left out on purpose: `_person_properties` adds it to PostHog
+# reads only, and it has no LaunchDarkly counterpart to map.
PERSON_PROPERTIES = frozenset(
{"email", "email_domain", "role", "created_at", "country"}
)
diff --git a/docs/engineering/feature-flags.md b/docs/engineering/feature-flags.md
index 738cf9ef89..f2d1d765d0 100644
--- a/docs/engineering/feature-flags.md
+++ b/docs/engineering/feature-flags.md
@@ -6,6 +6,80 @@ LaunchDarkly's answer, and log every disagreement. Call sites are unaffected by
the choice; they read through `evaluate_feature_flag()` on the backend and
`useGetFlag` / `useFlagStatus` on the frontend.
+## One set of flags, separate reporting
+
+Flags are defined once, in the **prod** PostHog project. Dev reads those
+definitions, while every event and person they produce
+goes to the **dev** project, so prod's analytics never mix with dev's.
+
+Only the backend reads flag definitions. The browser gets its flags from the
+backend's `GET /api/feature-flags` (`NEXT_PUBLIC_FEATURE_FLAG_BACKEND=server`),
+which evaluates them through the same path as the backend's own reads. So the
+split below covers the browser's flags too, with no PostHog flags client in
+the browser.
+
+> **`server` mode arrives with #15383.** Until that PR is merged, the browser's
+> backend selector (`frontend/src/services/feature-flags/flag-backend.ts`)
+> knows only `launchdarkly`, `posthog` and `dual`, and treats any other value,
+> `server` included, as `launchdarkly`. Set `server` on dev only once #15383 is
+> deployed there; before that, the split covers the backend's reads only.
+
+| Setting | Prod | Dev |
+|---|---|---|
+| `POSTHOG_API_KEY` (backend) | prod project API key | dev project API key |
+| `POSTHOG_FLAGS_API_KEY` (backend) | blank | prod project API key |
+| `POSTHOG_SECRET_KEY` (backend) | prod Feature Flags Secure API Key | prod Feature Flags Secure API Key |
+| `NEXT_PUBLIC_POSTHOG_KEY` (Vercel) | prod project API key | dev project API key |
+
+This applies to flags PostHog answers, so dev needs `FEATURE_FLAG_BACKEND=posthog`
+(under `dual`, LaunchDarkly's answer is served and PostHog's is only compared),
+and `NEXT_PUBLIC_FEATURE_FLAG_BACKEND=server` (once #15383 is deployed) so the
+browser asks the backend.
+
+A blank flags key reads flags from the analytics project, which is what prod
+and every self-hosted deployment do. `POSTHOG_SECRET_KEY` always belongs to the
+project flags are read from. All projects must be in the one region
+`POSTHOG_HOST` names.
+
+**Backend.** The flag client uses the flags key and the secret key, so it polls
+prod's definitions and evaluates in-process. When the flags key names another
+project it is built with `send=False`, so its `$feature_flag_called` events
+never leave the process, and every read is local only
+(`only_evaluate_locally`): a flag local evaluation cannot answer serves its
+default rather than asking prod's `/flags` with the user's email. That makes
+the secret key required on dev; without it every read serves its default, and
+the client logs a warning at startup. The analytics client
+(`backend/util/posthog_client.py`) keeps `POSTHOG_API_KEY`.
+
+This covers every PostHog flag read the backend makes: its own
+(`evaluate_feature_flag`, `get_feature_flag_value`, route gates) and the
+browser's through `/api/feature-flags`.
+
+**Browser.** In `server` mode the browser asks the backend and nothing else
+for its flags, so on dev it gets prod's definitions with no change of its own.
+posthog-js still talks only to the analytics project. The other
+`NEXT_PUBLIC_FEATURE_FLAG_BACKEND` values read flags in the browser itself:
+`launchdarkly` (the default) from LaunchDarkly, and `posthog` / `dual` from the
+project `NEXT_PUBLIC_POSTHOG_KEY` names. So dev must run `server` mode to share
+prod's flags. The one browser read left is `useExperiment`, which reads posthog-js and
+so, on dev, the dev project's experiment flags, until experiment arms are
+assigned on the backend.
+
+**Rolling out to dev only.** Every PostHog flag read the backend makes, its own
+and the browser's in `server` mode, carries the person property `environment`,
+the backend's `APP_ENV`: `local`, `dev` or `prod`, the same value the analytics
+events carry. A release condition `environment = dev` turns a flag on for dev
+alone; prod never matches it. A Vercel preview in `server` mode gets the answer
+of the backend it calls. The property is only an evaluation input: it is never
+stored on a person.
+
+Browser reads through posthog-js (`posthog` / `dual` mode, which is prod today)
+don't carry `environment`: they send `buildFlagPersonProperties` (email, email
+domain, role, created at) only. So use `environment` only in positive
+dev-only conditions like `environment = dev`. A condition like
+`environment = prod` or "`environment` is not `dev`" would turn on in prod's
+backend reads but stay off in prod's browser until prod runs `server` mode.
+
## Sharing PostHog's flag definitions
PostHog evaluates flags locally, in the process, from a set of flag definitions