AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
revert(backend): read PostHog flags from each deployment's own project again (#15572)
Reinier van der Leer06d623e5e04c
diff --git a/autogpt_platform/backend/.env.default b/autogpt_platform/backend/.env.defaultindex 3e617341ad..4eacd7b4d4 100644--- a/autogpt_platform/backend/.env.default+++ b/autogpt_platform/backend/.env.default@@ -390,15 +390,9 @@ ZEROBOUNCE_API_KEY=# Get API key from https://posthog.com - Project Settings > Project API KeyPOSTHOG_API_KEY=POSTHOG_HOST=https://eu.i.posthog.com-# Project API key of the project feature flags are read from, when that is not-# the analytics project above. Dev sets prod's key here (one set of flags) while-# its events stay in the dev project. Blank reads flags from POSTHOG_API_KEY's.-POSTHOG_FLAGS_API_KEY=-# Feature Flags Secure API Key of the project flags are read from, only for-# local feature-flag evaluation (project settings > Feature Flags). A personal-# API key also works, but PostHog is deprecating that use. Without it flag reads-# go over the wire, or serve their defaults when POSTHOG_FLAGS_API_KEY names-# another project.+# Feature Flags Secure API Key, only for local feature-flag evaluation (project+# settings > Feature Flags). A personal API key also works, but PostHog is+# deprecating that use. Without it flag reads go over the wire.POSTHOG_SECRET_KEY=# Where flag definitions are shared between processes: redis (default; one# elected refresher, every other process reads its copy), memory (process-local),diff --git a/autogpt_platform/backend/backend/util/feature_flag/__init__.py b/autogpt_platform/backend/backend/util/feature_flag/__init__.pyindex ba70029d39..166cb50580 100644--- a/autogpt_platform/backend/backend/util/feature_flag/__init__.py+++ b/autogpt_platform/backend/backend/util/feature_flag/__init__.py@@ -738,9 +738,9 @@ def _launchdarkly_flag_keys() -> frozenset[str] | None:def _person_properties(context: Context) -> dict[str, Any]:"""PostHog person properties for one read: the user's, plus ``environment``.- ``environment`` rides on every read, anonymous ones included: dev reads- prod's flag definitions, so a rollout to dev alone is a release condition- on it. It is the app env the analytics events also carry.+ ``environment`` rides on every read, anonymous ones included, so a release+ condition can tell deployments apart. It is the app env the analytics events+ also carry."""return {**_context_properties(context),diff --git a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py b/autogpt_platform/backend/backend/util/feature_flag/backends_test.pyindex 6428c3a79c..6db05b8436 100644--- a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py+++ b/autogpt_platform/backend/backend/util/feature_flag/backends_test.py@@ -377,8 +377,7 @@ class TestPostHogBackend:async def test_a_dev_only_rollout_evaluates_in_process(self, mocker, user_context, app_env, expected):- """Dev reads prod's definitions; a release condition on ``environment``- is how a flag goes out to dev alone."""+ """A release condition on ``environment`` decides in process, per app env."""use_backend(mocker, FeatureFlagBackend.POSTHOG)mocker.patch.object(ff.settings.config, "app_env", app_env)posthog_client = _posthog_rolled_out_to_dev_only()diff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog.py b/autogpt_platform/backend/backend/util/feature_flag/posthog.pyindex a3cf31dd37..14e785ca9e 100644--- a/autogpt_platform/backend/backend/util/feature_flag/posthog.py+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog.py@@ -46,22 +46,7 @@ _shut_down = Falsedef is_configured() -> bool:"""Whether PostHog has the project key a flag read needs."""- return bool(_flag_project_key())---def reads_another_project() -> bool:- """Whether flags come from a PostHog project other than the analytics one.-- Dev reads prod's flag definitions, so nothing a read does may create data- there: no ``$feature_flag_called`` events, and no remote ``/flags`` call- carrying a user's properties when local evaluation cannot answer.- """- flags_key = settings.secrets.posthog_flags_api_key- return bool(flags_key) and flags_key != settings.secrets.posthog_api_key---def _flag_project_key() -> str:- return settings.secrets.posthog_flags_api_key or settings.secrets.posthog_api_key+ return bool(settings.secrets.posthog_api_key)async def evaluate_flag(@@ -98,7 +83,6 @@ async def evaluate_flag(flag_key,distinct_id,person_properties=person_properties or None,- only_evaluate_locally=reads_another_project(),send_feature_flag_events=exposure,),)@@ -227,33 +211,22 @@ def _build_client() -> Posthog | None:return Nonesecret_key = settings.secrets.posthog_secret_key- separate_project = reads_another_project()- if separate_project and not secret_key:- logger.warning(- "POSTHOG_FLAGS_API_KEY names another PostHog project but "- "POSTHOG_SECRET_KEY is unset: with no local definitions and no remote "- "fallback, every flag read serves its default"- )# One shared refresher, so the definitions bill stops scaling with replica# count. Without a provider the SDK polls PostHog once per process.definition_cache = get_flag_definition_cache() if secret_key else None_definition_cache = definition_cache_client = Posthog(- _flag_project_key(),+ settings.secrets.posthog_api_key,host=settings.secrets.posthog_host,secret_key=secret_key or None,enable_local_evaluation=bool(secret_key),poll_interval=refresh_interval_seconds(),flag_definition_cache_provider=definition_cache,- # Its only events are $feature_flag_called, which would land in the- # flag project; this deployment's analytics belong in its own.- send=not separate_project,)_init_attempted = Truelogger.info("PostHog feature flag client initialized "- f"(flag project: {'separate' if separate_project else 'analytics'}, "- f"local evaluation: {'on' if secret_key else 'off'}, "+ f"(local evaluation: {'on' if secret_key else 'off'}, "f"definition cache: {type(definition_cache).__name__ if definition_cache else 'off'})")return _clientdiff --git a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.pyindex c6e637eb45..60e9f7d34a 100644--- a/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py+++ b/autogpt_platform/backend/backend/util/feature_flag/posthog_test.py@@ -11,10 +11,6 @@ from posthog.types import FeatureFlagResultimport backend.util.feature_flag.posthog as ph-# Taken at import, before any fixture can stub it: the test below needs the-# SDK's own capture path to show that send=False keeps events off the wire.-_REAL_CAPTURE = Posthog.capture-@pytest.fixture(autouse=True)def fresh_client(mocker):@@ -24,10 +20,9 @@ def fresh_client(mocker):mocker.patch.object(ph, "_shut_down", False)-def configure(mocker, *, api_key="phc_test", secret_key="", flags_key=""):+def configure(mocker, *, api_key="phc_test", secret_key=""):mocker.patch.object(ph.settings.secrets, "posthog_api_key", api_key)mocker.patch.object(ph.settings.secrets, "posthog_secret_key", secret_key)- mocker.patch.object(ph.settings.secrets, "posthog_flags_api_key", flags_key)def answer(*, value, payload=None):@@ -154,128 +149,6 @@ class TestClientConstruction:assert ph.get_flag_client() is not None-class TestTheFlagProject:- """Dev reads prod's flag definitions while its analytics stay in dev."""-- def test_without_a_flags_key_flags_come_from_the_analytics_project(self, mocker):- configure(mocker, api_key="phc_analytics")- posthog = mocker.patch.object(ph, "Posthog")-- ph.get_flag_client()-- args, kwargs = posthog.call_args- assert args[0] == "phc_analytics"- assert kwargs["send"] is True- assert ph.reads_another_project() is False-- def test_the_same_key_twice_is_one_project(self, mocker):- configure(mocker, api_key="phc_prod", flags_key="phc_prod")- posthog = mocker.patch.object(ph, "Posthog")-- ph.get_flag_client()-- args, kwargs = posthog.call_args- assert args[0] == "phc_prod"- assert kwargs["send"] is True-- def test_a_flags_key_reads_that_project_and_sends_it_nothing(self, mocker):- configure(- mocker, api_key="phc_dev", flags_key="phc_prod", secret_key="phs_prod"- )- posthog = mocker.patch.object(ph, "Posthog")-- ph.get_flag_client()-- args, kwargs = posthog.call_args- assert args[0] == "phc_prod"- assert kwargs["secret_key"] == "phs_prod"- assert kwargs["send"] is False-- def test_a_flags_key_alone_configures_flag_reads(self, mocker):- configure(mocker, api_key="", flags_key="phc_prod")-- assert ph.is_configured() is True- assert ph.reads_another_project() is True-- def test_a_separate_project_without_the_secret_key_warns(self, mocker):- configure(mocker, api_key="phc_dev", flags_key="phc_prod")- mocker.patch.object(ph, "Posthog")- warn = mocker.patch.object(ph.logger, "warning")-- ph.get_flag_client()-- assert "POSTHOG_SECRET_KEY" in warn.call_args.args[0]-- @pytest.mark.asyncio- @pytest.mark.parametrize(- "flags_key,only_locally", [("", False), ("phc_test", False), ("phc_prod", True)]- )- async def test_only_a_separate_project_is_read_locally_only(- self, mocker, flags_key, only_locally- ):- configure(mocker, flags_key=flags_key)- client = mocker.Mock()- client.get_feature_flag_result.return_value = answer(value=True)- mocker.patch.object(ph, "get_flag_client", return_value=client)-- await ph.evaluate_flag("hire-experts", "u-1")-- _, kwargs = client.get_feature_flag_result.call_args- assert kwargs["only_evaluate_locally"] is only_locally---class TestASeparateFlagProjectThroughTheRealSDK:- """What the installed posthog-python does with the client built above."""-- @pytest.mark.asyncio- @pytest.mark.parametrize(- "flags_key,asks_remotely", [("", True), ("phc_prod", False)]- )- async def test_a_flag_missing_locally_goes_remote_only_within_one_project(- self, mocker, flags_key, asks_remotely- ):- """The remote fallback would hand the flag project a user's email."""- # The flag-called events are not what this test is about.- mocker.patch.object(Posthog, "capture")- client, remote = _real_client(mocker, flags_key=flags_key)- try:- value = await ph.evaluate_flag(- "not-in-the-definitions", "u-1", {"email": "[email protected]"}- )- finally:- ph.shutdown_posthog_flags()-- assert value == (None, False)- assert remote.called is asks_remotely-- @pytest.mark.asyncio- async def test_reads_from_a_separate_project_send_it_no_events(self, mocker):- mocker.patch.object(Posthog, "capture", _REAL_CAPTURE)- captured = mocker.spy(Posthog, "capture")- uploads = [- mocker.patch("posthog.consumer.batch_post"),- mocker.patch("posthog.consumer._send_v1_batch"),- ]- client, remote = _real_client(mocker, flags_key="phc_prod")- try:- # Exposure reads, the only ones that form $feature_flag_called.- assert await ph.evaluate_flag("hire-experts", "u-1", exposure=True) == (- True,- True,- )- assert await ph.evaluate_flag(- "not-in-the-definitions", "u-1", exposure=True- ) == (None, False)- client.flush()- finally:- ph.shutdown_posthog_flags()-- # The SDK still forms the events; none is uploaded.- assert captured.called- assert not remote.called- assert not any(upload.called for upload in uploads)--class TestInMemoryRead:def test_definitions_not_loaded_yet_are_never_fetched(self, mocker):"""Fetching them first is the SDK's default, and a network call per read."""@@ -451,26 +324,6 @@ class TestRawRead:loop.close()-def _real_client(mocker, *, flags_key: str):- """A real flag client holding one rolled-out flag, with /flags recorded."""- configure(mocker, api_key="phc_dev", flags_key=flags_key, secret_key="phs_test")- mocker.patch.object(ph, "get_flag_definition_cache", return_value=None)- client = ph.get_flag_client()- assert client is not None- client.feature_flags = [- {- "id": 1,- "key": "hire-experts",- "active": True,- "filters": {"groups": [{"properties": [], "rollout_percentage": 100}]},- }- ]- remote = mocker.patch.object(- client, "_get_flags_decision", side_effect=ConnectionError("offline")- )- return client, remote--def _rolled_out(key: str, percentage: int, payload: str | None = None) -> dict:filters: dict = {"groups": [{"properties": [], "rollout_percentage": percentage}]}if payload is not None:diff --git a/autogpt_platform/backend/backend/util/settings.py b/autogpt_platform/backend/backend/util/settings.pyindex 98988abe9f..c8a7ee8ab2 100644--- a/autogpt_platform/backend/backend/util/settings.py+++ b/autogpt_platform/backend/backend/util/settings.py@@ -1417,20 +1417,12 @@ class Secrets(UpdateTrackingModel["Secrets"], BaseSettings):posthog_host: str = Field(default="https://eu.i.posthog.com", description="PostHog host URL")- posthog_flags_api_key: str = Field(- default="",- description="Project API key of the PostHog project feature flags are read "- "from, when that is not the analytics project. Dev sets prod's, so both "- "share one set of flag definitions while dev's events stay in its own "- "project. Blank reads flags from the POSTHOG_API_KEY project.",- )posthog_secret_key: str = Field(default="",description="PostHog Feature Flags Secure API Key (project settings > "- "Feature Flags) of the project flags are read from, for local flag "- "evaluation; without it flag reads fall back to a remote /flags call, or "- "serve their defaults when POSTHOG_FLAGS_API_KEY names another project. "- "A personal API key also works, but PostHog is deprecating that use.",+ "Feature Flags), for local flag evaluation; without it flag reads fall back "+ "to a remote /flags call. A personal API key also works, but PostHog is "+ "deprecating that use.",)# Add more secret fields as neededdiff --git a/docs/engineering/feature-flags.md b/docs/engineering/feature-flags.mdindex 1dd1224d2f..02d66c8210 100644--- a/docs/engineering/feature-flags.md+++ b/docs/engineering/feature-flags.md@@ -6,79 +6,48 @@ LaunchDarkly's answer, and log every disagreement. Call sites are unaffected bythe choice; they read through `evaluate_feature_flag()` on the backend and`useGetFlag` / `useFlagStatus` on the frontend.-## One set of flags, separate reporting+## One PostHog project per environment-Flags are defined once, in the **prod** PostHog project. Dev reads those-definitions, while every event and person they produce-goes to the **dev** project, so prod's analytics never mix with dev's.+Dev and Prod each have their own PostHog project, and each project holds its+own flags. A deployment reads its flags from the same project its analytics go+to, the one `POSTHOG_API_KEY` names, and evaluates them locally with that+project's Feature Flags Secure API Key in `POSTHOG_SECRET_KEY`. Where local+evaluation can't decide a read, the SDK asks PostHog's `/flags` endpoint+instead, unless the read is local-only, as a signed-out visitor's own read is.-Only the backend reads flag definitions. The browser gets its flags from the-backend's `GET /api/feature-flags` (`NEXT_PUBLIC_FEATURE_FLAG_BACKEND=server`),-which evaluates them through the same path as the backend's own reads. So the-split below covers the browser's flags too, with no PostHog flags client in-the browser.--> **`server` mode arrives with #15383.** Until that PR is merged, the browser's-> backend selector (`frontend/src/services/feature-flags/flag-backend.ts`)-> knows only `launchdarkly`, `posthog` and `dual`, and treats any other value,-> `server` included, as `launchdarkly`. Set `server` on dev only once #15383 is-> deployed there; before that, the split covers the backend's reads only.--| Setting | Prod | Dev |+| | Dev | Prod ||---|---|---|-| `POSTHOG_API_KEY` (backend) | prod project API key | dev project API key |-| `POSTHOG_FLAGS_API_KEY` (backend) | blank | prod project API key |-| `POSTHOG_SECRET_KEY` (backend) | prod Feature Flags Secure API Key | prod Feature Flags Secure API Key |-| `NEXT_PUBLIC_POSTHOG_KEY` (Vercel) | prod project API key | dev project API key |--This applies to flags PostHog answers, so dev needs `FEATURE_FLAG_BACKEND=posthog`-(under `dual`, LaunchDarkly's answer is served and PostHog's is only compared),-and `NEXT_PUBLIC_FEATURE_FLAG_BACKEND=server` (once #15383 is deployed) so the-browser asks the backend.--A blank flags key reads flags from the analytics project, which is what prod-and every self-hosted deployment do. `POSTHOG_SECRET_KEY` always belongs to the-project flags are read from. All projects must be in the one region-`POSTHOG_HOST` names.--**Backend.** The flag client uses the flags key and the secret key, so it polls-prod's definitions and evaluates in-process. When the flags key names another-project it is built with `send=False`, so its `$feature_flag_called` events-never leave the process, and every read is local only-(`only_evaluate_locally`): a flag local evaluation cannot answer serves its-default rather than asking prod's `/flags` with the user's email. That makes-the secret key required on dev; without it every read serves its default, and-the client logs a warning at startup. The analytics client-(`backend/util/posthog_client.py`) keeps `POSTHOG_API_KEY`.--This covers every PostHog flag read the backend makes: its own-(`evaluate_feature_flag`, `get_feature_flag_value`, route gates) and the-browser's through `/api/feature-flags`.--**Browser.** In `server` mode the browser asks the backend and nothing else-for its flags, so on dev it gets prod's definitions with no change of its own.-posthog-js still talks only to the analytics project. The other-`NEXT_PUBLIC_FEATURE_FLAG_BACKEND` values read flags in the browser itself:-`launchdarkly` (the default) from LaunchDarkly, and `posthog` / `dual` from the-project `NEXT_PUBLIC_POSTHOG_KEY` names. So dev must run `server` mode to share-prod's flags. The one browser read left is `useExperiment`, which reads posthog-js and-so, on dev, the dev project's experiment flags, until experiment arms are-assigned on the backend.--**Rolling out to dev only.** Every PostHog flag read the backend makes, its own-and the browser's in `server` mode, carries the person property `environment`,-the backend's `APP_ENV`: `local`, `dev` or `prod`, the same value the analytics-events carry. A release condition `environment = dev` turns a flag on for dev-alone; prod never matches it. A Vercel preview in `server` mode gets the answer-of the backend it calls. The property is only an evaluation input: it is never-stored on a person.--Browser reads through posthog-js (`posthog` / `dual` mode, which is prod today)-don't carry `environment`: they send `buildFlagPersonProperties` (email, email-domain, role, created at) only. So use `environment` only in positive-dev-only conditions like `environment = dev`. A condition like-`environment = prod` or "`environment` is not `dev`" would turn on in prod's-backend reads but stay off in prod's browser until prod runs `server` mode.+| PostHog project | [281733](https://eu.posthog.com/project/281733/feature_flags) | [117338](https://eu.posthog.com/project/117338/feature_flags) |+| `POSTHOG_API_KEY`, `POSTHOG_SECRET_KEY` (backend) | Dev project's | Prod project's |+| `NEXT_PUBLIC_POSTHOG_KEY` (Vercel) | Dev project's | Prod project's |++PostHog has no environments within a project, so a flag both deployments read+is defined in each project. The **Projects** tab on the+[flag list](https://eu.posthog.com/project/281733/feature_flags?tab=projects)+and on each flag's page shows a flag across both projects side by side.++**Until Prod switches to PostHog**, Prod serves LaunchDarkly's answers+(`FEATURE_FLAG_BACKEND=dual` only compares PostHog's against them), so a change+to a Prod flag is made in LaunchDarkly and mirrored in the Prod PostHog project.++**The browser.** With `NEXT_PUBLIC_FEATURE_FLAG_BACKEND=server` the browser+gets its flags from the backend's `GET /api/feature-flags`, which evaluates+them through the same path as the backend's own reads, so the browser and the+backend give one answer from one project. A Vercel preview in `server` mode+gets the answer of the backend it calls. The other values read flags in the+browser itself: `launchdarkly` (the default) from LaunchDarkly, and `posthog` /+`dual` from the project `NEXT_PUBLIC_POSTHOG_KEY` names. `useExperiment` reads+posthog-js in every mode.++**The `environment` property.** Every PostHog flag read the backend makes, its+own and the browser's in `server` mode, carries the person property+`environment`: the backend's `APP_ENV` (`local`, `dev` or `prod`), the same+value its analytics events carry. A release condition can use it in either+project like any other person property. It is only an evaluation input, never+stored on a person. Browser reads through posthog-js (`posthog` / `dual` mode)+don't carry it: they send `buildFlagPersonProperties` (email, email domain,+role, created at) only, so a condition on `environment` matches only the+backend's reads there.**Splitting signed-out visitors.** A visitor who allowed analytics is bucketedunder `visitor:<their PostHog id>`, but only by conditions their read can@@ -173,5 +142,6 @@ process, `cached` once per poll on each of the other N−1, and `empty` and| The refresher dies | Its lock expires and another process takes the job on its next poll. || A definitions fetch fails | The definitions already in memory stay in use, and the refresher keeps writing that copy to the cache. |-Nothing here can make a flag read wait on Redis or on PostHog: flag evaluation-only ever reads the definitions already in the process.+Nothing here can make a flag read wait on Redis: evaluation reads the+definitions already in the process, and only a read they can't decide waits on+PostHog's `/flags`.