AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
fix(platform): PostHog flag reads carry email, and dual mode logs only real disagreements (#15357)
Reinier van der Leer4473fbbcb195
diff --git a/autogpt_platform/backend/backend/util/feature_flag/__init__.py b/autogpt_platform/backend/backend/util/feature_flag/__init__.pyindex 18936f41a8..3c49d323c0 100644--- a/autogpt_platform/backend/backend/util/feature_flag/__init__.py+++ b/autogpt_platform/backend/backend/util/feature_flag/__init__.py@@ -15,6 +15,7 @@ from autogpt_libs.auth.dependencies import get_optional_user_idfrom fastapi import HTTPException, Securityfrom ldclient import Context, LDClientfrom ldclient.config import Config+from ldclient.evaluation import EvaluationDetailfrom typing_extensions import ParamSpecfrom backend.util.cache import cached@@ -40,8 +41,8 @@ _init_attempted = False_shadow_evaluations: set[asyncio.Task] = set()_shadow_evaluations_stopped = FalseMAX_CONCURRENT_SHADOW_EVALUATIONS = 100-# Flags PostHog could not answer for, already reported once. Before phase 2-# creates the flags that is every read; one record per flag keeps the signal.+# Flags PostHog could not answer while LaunchDarkly could, already reported+# once: such a flag fails every read, so one record per flag keeps the signal._unanswered_flags_reported: set[str] = set()@@ -210,38 +211,6 @@ class Flag(str, Enum):# a cohort before it reaches everyone.CHAT_CONNECTION_UPSELL = "chat-connection-upsell"- # Shrinks what Otto reads: strips builder-UI annotations from the- # block schemas, and digests oversized tool results to the workspace.---# LaunchDarkly keys whose targeting is segment membership or an individual-user-# target — neither of which any PostHog cohort reproduces until phase 2 creates-# one. From ld-targeting-summary.md; attribute rules (email domain, role,-# signup date) are not listed because `_person_properties` already carries them.-LD_UNPORTED_TARGETING = frozenset(- {- "ai-agent-execution-summary",- "artifacts",- "artifacts-page",- "autogpt-new-layout",- "AutoMod",- "beta-blocks",- "chat",- "chat-mode-option",- "chat-search",- "chat-sharing",- "copilot-bot-platforms",- "dream-pass-enabled",- "enable-platform-payment",- "generic-trigger-agents",- "graphiti-memory",- "new-tool-ui",- "nightly-copilot",- "SHOW_ORG_SETTINGS",- "task-progress-bar",- }-)-def initialize_feature_flags() -> None:"""Start whichever vendor(s) the configured backend reads from."""@@ -614,11 +583,12 @@ async def _record_mismatch(logger.warning(f"PostHog shadow evaluation raised for {flag_key}: {e}")return- if ld_result == ph_result:- return-ld_value, ld_evaluated = ld_resultph_value, ph_evaluated = ph_result+ # A mismatch is a read whose served value would change at the switch, or+ # one PostHog cannot answer where LaunchDarkly can.+ if ph_value == ld_value and (ph_evaluated or not ld_evaluated):+ returnif not ph_evaluated:if flag_key in _unanswered_flags_reported:return@@ -631,11 +601,6 @@ async def _record_mismatch("user": _user_digest(user_id),"launchdarkly": {"value": ld_value, "evaluated": ld_evaluated},"posthog": {"value": ph_value, "evaluated": ph_evaluated},- # Segment and individual-user targeting has no PostHog cohort- # until phase 2 builds one, so these flags are expected to- # disagree — the diff-week report has to separate them from- # real divergence rather than drown in them.- "expected_until_cohorts_exist": flag_key in LD_UNPORTED_TARGETING,},default=repr,sort_keys=True,@@ -677,9 +642,10 @@ async def _evaluate_launchdarkly(``evaluated`` is False whenever *default* is standing in for an answerLaunchDarkly could not give — no client, an uninitialised one, a failed- user-context lookup, or an evaluation that raised. An initialised client- is not on its own enough: the context lookup is a database read, so a- live client can still fail to produce a value.+ user-context lookup, a flag LaunchDarkly does not have (or has switched+ off with no off variation), or an evaluation that raised. An initialised+ client is not on its own enough: the context lookup is a database read,+ so a live client can still fail to produce a value."""try:client = get_client()@@ -696,15 +662,18 @@ async def _evaluate_launchdarkly(user_id)- # Evaluate flag- result = client.variation(flag_key, user_context, default)+ detail = client.variation_detail(flag_key, user_context, default)logger.debug(- f"Feature flag {flag_key} for user {user_id}: {result} (type: {type(result).__name__})"+ "Feature flag %s for user %s: %r (%s)",+ flag_key,+ user_id,+ detail.value,+ detail.reason,)# A degraded context evaluates fine, it just answers for an anonymous# user rather than this one — so the value is a guess, not an answer.- return result, context_resolved+ return detail.value, context_resolved and not detail.is_default_value()except Exception as e:logger.warning(@@ -718,14 +687,15 @@ def _person_properties(context: Context) -> dict[str, Any]:Reads the LaunchDarkly context rather than the auth row so both vendorssee one cached lookup and cannot drift; ``custom.role`` is dropped- because PostHog targets flat properties, and ``email`` because no ported- rule reads it — individual targets key on the ``distinct_id``.+ because PostHog targets flat properties. ``email`` is sent because the+ ported email rules and the ``ld:employee`` cohort match on it, and a rule+ on a property the read omits cannot be evaluated in-process."""if context.anonymous:return {}return {attribute: context.get(attribute)- for attribute in ("role", "email_domain", "created_at", "country")+ for attribute in ("role", "email", "email_domain", "created_at", "country")if context.get(attribute) is not None}@@ -1072,11 +1042,14 @@ def _flag_backend_initialized() -> bool:@contextlib.contextmanagerdef mock_flag_variation(flag_key: str, return_value: Any):"""Context manager for testing feature flags."""- original_variation = get_client().variation- get_client().variation = lambda key, context, default: (- return_value if key == flag_key else original_variation(key, context, default)+ client = get_client()+ original = client.variation_detail+ client.variation_detail = lambda key, context, default: (+ EvaluationDetail(return_value, 0, {"kind": "FALLTHROUGH"})+ if key == flag_key+ else original(key, context, default))try:yieldfinally:- get_client().variation = original_variation+ client.variation_detail = originaldiff --git a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py b/autogpt_platform/backend/backend/util/feature_flag/backends_test.pyindex 77eaf1857c..c8e11bc15d 100644--- a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py+++ b/autogpt_platform/backend/backend/util/feature_flag/backends_test.py@@ -12,6 +12,7 @@ import sentry_sdk.feature_flagsfrom fastapi import HTTPExceptionfrom ldclient import Context, LDClientfrom ldclient.config import Config as LDConfig+from ldclient.evaluation import EvaluationDetailfrom ldclient.integrations.test_data import TestDatafrom posthog import Posthogfrom sentry_sdk.tracing import Span@@ -114,17 +115,17 @@ class TestDefaultBackendIsUnchanged:self, mocker, ld_client, user_context):posthog = mocker.patch.object(ph, "evaluate_flag")- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)- ld_client.variation.assert_called_once()+ ld_client.variation_detail.assert_called_once()posthog.assert_not_called()@pytest.mark.asyncioasync def test_the_default_reads_launchdarkly_for_raw_values(self, ld_client, user_context):- ld_client.variation.return_value = {"daily": 5}+ ld_client.variation_detail.return_value = served({"daily": 5})assert await ff.get_feature_flag_value("copilot-cost-limits", "system") == {"daily": 5@@ -231,7 +232,7 @@ class TestPostHogBackend:async def test_targeting_attributes_are_passed_as_person_properties(self, mocker):"""Every attribute an LD rule targets on has to reach PostHog, or theflag silently evaluates against a user without them — and nothing- more: the raw email is not one of them."""+ more: ``custom`` is LaunchDarkly's nesting of ``role``."""use_backend(mocker, FeatureFlagBackend.POSTHOG)client = stub_posthog(mocker, value=True)context = (@@ -255,10 +256,40 @@ class TestPostHogBackend:_, kwargs = client.evaluate_flags.call_argsassert kwargs["person_properties"] == {"role": "admin",+ "email": "[email protected]","email_domain": "agpt.co","created_at": "2026-05-07T12:00:00+00:00",}+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "email,expected", [("[email protected]", (True, True)), ("[email protected]", (False, True))]+ )+ async def test_an_email_cohort_evaluates_in_process(self, mocker, email, expected):+ """The synced ``ld:employee`` cohort matches on ``email``; without it a+ read cannot be decided locally and becomes a remote /flags call."""+ use_backend(mocker, FeatureFlagBackend.POSTHOG)+ posthog_client = _posthog_with_an_employee_cohort()+ mocker.patch.object(ph, "get_flag_client", return_value=posthog_client)+ context = (+ Context.builder("u-1")+ .kind("user")+ .anonymous(False)+ .set("email", email)+ .set("email_domain", email.split("@")[1])+ .build()+ )+ mocker.patch(+ "backend.util.feature_flag._fetch_user_context_status",+ return_value=(context, True),+ )+ try:+ result = await evaluate_feature_flag(Flag.COPILOT_VOICE_MODE, "u-1")+ finally:+ posthog_client.shutdown()++ assert result == expected+@pytest.mark.asyncioasync def test_an_anonymous_context_carries_no_person_properties(self, mocker):"""The `"system"`-keyed config flags have no user to describe."""@@ -296,7 +327,7 @@ class TestDualBackend:self, mocker, ld_client, user_context):use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)stub_posthog(mocker, value=False)assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)@@ -306,7 +337,7 @@ class TestDualBackend:self, mocker, ld_client, user_context, caplog):use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)stub_posthog(mocker, value=False)user_id = str(uuid.uuid4())@@ -333,7 +364,7 @@ class TestDualBackend:self, mocker, ld_client, user_context, caplog):use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)stub_posthog(mocker, value=True)with caplog.at_level(@@ -353,7 +384,7 @@ class TestDualBackend:"""Both say "off", but only one of them knows it — that is thedifference the diff week exists to find."""use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = False+ ld_client.variation_detail.return_value = served(False)stub_posthog(mocker, value=None)with caplog.at_level(@@ -380,7 +411,7 @@ class TestDualBackend:"""Before phase 2 creates the flags PostHog answers nothing, so everyread would otherwise log the same record."""use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)stub_posthog(mocker, value=None)with caplog.at_level(@@ -405,7 +436,7 @@ class TestDualBackend:self, mocker, ld_client, user_context, caplog):use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)stub_posthog(mocker, value=False)with caplog.at_level(@@ -426,12 +457,40 @@ class TestDualBackend:== 3)+ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "posthog_value", [None, True], ids=["posthog-lacks-it", "posthog-serves-it"]+ )+ async def test_a_flag_launchdarkly_lacks_logs_nothing_if_the_value_holds(+ self, mocker, user_context, caplog, posthog_value+ ):+ """LaunchDarkly serves the caller's default; a PostHog read serving the+ same changes nothing at the switch, answered or not."""+ use_backend(mocker, FeatureFlagBackend.DUAL)+ ld = _launchdarkly_with(TestData.data_source())+ mocker.patch("backend.util.feature_flag.ldclient.get", return_value=ld)+ stub_posthog(mocker, value=posthog_value)++ with caplog.at_level(+ logging.WARNING, logger="backend.util.feature_flag.mismatch"+ ):+ try:+ result = await evaluate_feature_flag(Flag.COPILOT_SDK, "u-1", True)+ await drain_shadow_evaluations()+ finally:+ ld.close()++ assert result == (True, False)+ assert not [+ r for r in caplog.records if r.name == "backend.util.feature_flag.mismatch"+ ]+@pytest.mark.asyncioasync def test_a_posthog_failure_cannot_break_the_read(self, mocker, ld_client, user_context):use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)mocker.patch.object(ph, "evaluate_flag", side_effect=Exception("boom"))assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is True@@ -443,7 +502,7 @@ class TestDualBackend:):"""A mismatch record must never be the thing that raises."""use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = object()+ ld_client.variation_detail.return_value = served(object())stub_posthog(mocker, value=True)with caplog.at_level(@@ -474,6 +533,39 @@ class TestDualBackend:assert ff._flag_backend_initialized() is False+class TestLaunchDarklyAnswers:+ """Through LaunchDarkly's real evaluator: only a served variation is an answer."""++ @pytest.mark.asyncio+ @pytest.mark.parametrize(+ "configure,expected",+ [+ (lambda td: td.update(td.flag("AutoMod").boolean_flag()), (True, True)),+ (+ lambda td: td.update(td.flag("AutoMod").boolean_flag().on(False)),+ (False, True),+ ),+ (lambda td: None, (True, False)),+ ],+ ids=["on", "switched-off", "absent"],+ )+ async def test_only_a_served_variation_is_authoritative(+ self, mocker, user_context, configure, expected+ ):+ """A switched-off flag still answers with its off variation; a flag+ LaunchDarkly does not have answers nothing, whatever the default."""+ td = TestData.data_source()+ configure(td)+ ld = _launchdarkly_with(td)+ mocker.patch("backend.util.feature_flag.ldclient.get", return_value=ld)+ try:+ result = await evaluate_feature_flag(Flag.AUTOMOD, "u-1", True)+ finally:+ ld.close()++ assert result == expected++class TestDualStaysOffTheRequestPath:"""The shadow answer is discarded, so nobody may wait for it."""@@ -482,7 +574,7 @@ class TestDualStaysOffTheRequestPath:self, mocker, ld_client, user_context):use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)posthog_started = asyncio.Event()async def slow_posthog(*args, **kwargs):@@ -506,7 +598,7 @@ class TestDualStaysOffTheRequestPath:"""The failed-lookup path is deliberately uncached, so evaluating thetwo vendors independently would double its database reads."""use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)stub_posthog(mocker, value=True)lookup = mocker.patch("backend.util.feature_flag._fetch_user_context_status",@@ -523,7 +615,7 @@ class TestDualStaysOffTheRequestPath:"""A slow PostHog costs the diff week its samples, not the processits memory."""use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)mocker.patch.object(ff, "MAX_CONCURRENT_SHADOW_EVALUATIONS", 2)async def never_finishes(*args, **kwargs):@@ -539,29 +631,6 @@ class TestDualStaysOffTheRequestPath:for task in list(ff._shadow_evaluations):task.cancel()- @pytest.mark.asyncio- async def test_segment_targeted_flags_are_marked_as_expected(- self, mocker, ld_client, user_context, caplog- ):- """Until phase 2 builds the cohorts these disagree by construction, and- the diff-week report has to be able to set them aside."""- use_backend(mocker, FeatureFlagBackend.DUAL)- ld_client.variation.return_value = True- stub_posthog(mocker, value=False)-- with caplog.at_level(- logging.WARNING, logger="backend.util.feature_flag.mismatch"- ):- await evaluate_feature_flag(Flag.GRAPHITI_MEMORY, "u-1")- await drain_shadow_evaluations()-- [message] = [- r.getMessage()- for r in caplog.records- if r.name == "backend.util.feature_flag.mismatch"- ]- assert _mismatch_record(message)["expected_until_cohorts_exist"] is True-class TestShutdownStopsShadowEvaluations:"""A shadow read that outlives teardown rebuilds the client it just closed:@@ -581,7 +650,7 @@ class TestShutdownStopsShadowEvaluations:use_backend(mocker, FeatureFlagBackend.DUAL)mocker.patch.object(ff, "shutdown_launchdarkly")mocker.patch.object(ph, "shutdown_posthog_flags")- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)async def never_finishes(*args, **kwargs):await asyncio.sleep(30)@@ -602,7 +671,7 @@ class TestShutdownStopsShadowEvaluations:use_backend(mocker, FeatureFlagBackend.DUAL)mocker.patch.object(ff, "shutdown_launchdarkly")mocker.patch.object(ph, "shutdown_posthog_flags")- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)get_client = mocker.patch.object(ph, "get_flag_client")ff.shutdown_feature_flags()@@ -710,7 +779,7 @@ class TestSentryFlagContext:self, mocker, ld_client, user_context, sentry_flags, backend):use_backend(mocker, backend)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)stub_posthog(mocker, value=backend is FeatureFlagBackend.POSTHOG)assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is True@@ -722,7 +791,7 @@ class TestSentryFlagContext:self, ld_client, user_context, sentry_flags, initialized):ld_client.is_initialized.return_value = initialized- ld_client.variation.return_value = {"daily": 5}+ ld_client.variation_detail.return_value = served({"daily": 5})await ff.get_feature_flag_value("copilot-cost-limits", "u-1", {"daily": 1})assert sentry_flags.get() == []@@ -746,7 +815,7 @@ class TestSentryFlagContext:async def test_a_non_boolean_answer_to_a_boolean_read_records_the_default(self, ld_client, user_context, sentry_flags):- ld_client.variation.return_value = "on"+ ld_client.variation_detail.return_value = served("on")assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1", True) == (True,@@ -781,7 +850,7 @@ class TestSentryFlagContext:use_backend(mocker, FeatureFlagBackend.LAUNCHDARKLY)mocker.patch("backend.util.feature_flag.is_configured", return_value=configured)ld_client.is_initialized.return_value = initialized- ld_client.variation.return_value = answer+ ld_client.variation_detail.return_value = served(answer)# A stale earlier answer must not be what an error from this route carries.sentry_sdk.feature_flags.add_feature_flag(Flag.HIRE_EXPERTS.value, True)@@ -820,7 +889,7 @@ class TestSentryFlagContext:sentry_sdk.feature_flags.add_feature_flag(f"{Flag.HIRE_EXPERTS.value}.fallback", True)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1")@@ -838,7 +907,7 @@ class TestSentryFlagContext:"add_feature_flag",side_effect=RuntimeError("sentry down"),)- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)@@ -889,6 +958,10 @@ def _launchdarkly_serving_all_but_india() -> LDClient:.if_not_match("country", "IN").then_return(1))+ return _launchdarkly_with(td)+++def _launchdarkly_with(td: TestData) -> LDClient:return LDClient(LDConfig("sdk-test", update_processor_class=td, send_events=False))@@ -929,6 +1002,46 @@ def _posthog_serving_all_but_india() -> Posthog:return client+def _posthog_with_an_employee_cohort() -> Posthog:+ """The shape the sync gives an email-targeted segment, evaluated locally."""+ client = Posthog("phc-test", secret_key="phx-test", enable_local_evaluation=False)+ client.feature_flags = [+ {+ "id": 1,+ "key": Flag.COPILOT_VOICE_MODE.value,+ "active": True,+ "filters": {+ "groups": [+ {+ "properties": [{"key": "id", "type": "cohort", "value": 7}],+ "rollout_percentage": 100,+ }+ ]+ },+ }+ ]+ client.cohorts = {+ "7": {+ "type": "OR",+ "values": [+ {+ "type": "AND",+ "values": [+ {+ "key": "email",+ "type": "person",+ "operator": "regex",+ "value": r"@agpt\.co$",+ }+ ],+ }+ ],+ }+ }+ client._get_flags_decision = _no_remote_flags+ return client++def _no_remote_flags(*args, **kwargs):raise ConnectionError("remote /flags is unavailable in tests")@@ -941,3 +1054,8 @@ def _gated_route():return "served"return route+++def served(value):+ """LaunchDarkly's answer for a flag it has, as its fallthrough serves it."""+ return EvaluationDetail(value, 0, {"kind": "FALLTHROUGH"})diff --git a/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py b/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.pyindex cc6381f3ff..365aa1b1e1 100644--- a/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py+++ b/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py@@ -7,6 +7,7 @@ import uuidimport pytestfrom ldclient import Context, LDClient+from ldclient.evaluation import EvaluationDetailfrom posthog import Posthogfrom posthog.request import GetResponsefrom pydantic import ValidationError@@ -418,7 +419,7 @@ class TestDefaultBackendIsUntouched:self, mocker, ld_client, user_context):build = mocker.patch.object(ph, "get_flag_definition_cache")- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)build.assert_not_called()@@ -562,3 +563,8 @@ class TestMetrics:provider(redis).get_flag_definitions()assert REGISTRY.get_sample_value(name, {"outcome": "cached"}) == before + 1+++def served(value):+ """LaunchDarkly's answer for a flag it has, as its fallthrough serves it."""+ return EvaluationDetail(value, 0, {"kind": "FALLTHROUGH"})diff --git a/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.py b/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.pyindex 3b57d14d79..66222233f1 100644--- a/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.py+++ b/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.py@@ -8,6 +8,7 @@ import pytestfrom fastapi import HTTPExceptionfrom ldclient import Context, LDClientfrom ldclient.config import Config+from ldclient.evaluation import EvaluationDetailfrom ldclient.integrations.test_data import TestDataimport backend@@ -39,7 +40,7 @@ def ld_client(mocker):@pytest.mark.asyncioasync def test_feature_flag_enabled(ld_client):- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)@feature_flag("test-flag")async def test_function(user_id: str):@@ -47,12 +48,12 @@ async def test_feature_flag_enabled(ld_client):result = await test_function(user_id="test-user")assert result == "success"- ld_client.variation.assert_called_once()+ ld_client.variation_detail.assert_called_once()@pytest.mark.asyncioasync def test_feature_flag_unauthorized_response(ld_client):- ld_client.variation.return_value = False+ ld_client.variation_detail.return_value = served(False)@feature_flag("test-flag")async def test_function(user_id: str):@@ -65,23 +66,23 @@ async def test_feature_flag_unauthorized_response(ld_client):def test_mock_flag_variation(ld_client):with mock_flag_variation("test-flag", True):- assert ld_client.variation("test-flag", None, False) is True+ assert ld_client.variation_detail("test-flag", None, False).value is Truewith mock_flag_variation("test-flag", False):- assert ld_client.variation("test-flag", None, True) is False+ assert ld_client.variation_detail("test-flag", None, True).value is False@pytest.mark.asyncioasync def test_is_feature_enabled(ld_client):"""Test the is_feature_enabled helper function."""ld_client.is_initialized.return_value = True- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)result = await is_feature_enabled(Flag.AUTOMOD, "user123", default=False)assert result is True- ld_client.variation.assert_called_once()- call_args = ld_client.variation.call_args+ ld_client.variation_detail.assert_called_once()+ call_args = ld_client.variation_detail.call_argsassert call_args[0][0] == "AutoMod" # flag_keyassert call_args[0][2] is False # default value@@ -94,7 +95,7 @@ async def test_is_feature_enabled_not_initialized(ld_client):result = await is_feature_enabled(Flag.AUTOMOD, "user123", default=True)assert result is True # Should return default- ld_client.variation.assert_not_called()+ ld_client.variation_detail.assert_not_called()@pytest.mark.asyncio@@ -403,13 +404,13 @@ class TestEnvOverrideWiring:return "success"assert await test_function(user_id="test-user") == "success"- ld_client.variation.assert_not_called()+ ld_client.variation_detail.assert_not_called()@pytest.mark.asyncioasync def test_decorator_per_flag_false_wins_over_launchdarkly(self, ld_client, monkeypatch: pytest.MonkeyPatch):- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)monkeypatch.setenv("FORCE_FLAG_TEST_FLAG", "false")@feature_flag("test-flag")@@ -419,7 +420,7 @@ class TestEnvOverrideWiring:with pytest.raises(HTTPException) as exc_info:await test_function(user_id="test-user")assert exc_info.value.status_code == 404- ld_client.variation.assert_not_called()+ ld_client.variation_detail.assert_not_called()@pytest.mark.asyncioasync def test_dependency_force_all_wins_over_unconfigured_sdk(@@ -456,7 +457,7 @@ class TestEnvOverrideWiring:explicit passthrough every request to a disabled route files anERROR and buries real flag-evaluation failures."""- ld_client.variation.return_value = False+ ld_client.variation_detail.return_value = served(False)@feature_flag("test-flag")async def test_function(user_id: str):@@ -496,7 +497,7 @@ class TestEnvOverrideWiring:"""mocker.patch("backend.util.feature_flag.is_configured", return_value=True)ld_client.is_initialized.return_value = True- ld_client.variation.return_value = False+ ld_client.variation_detail.return_value = served(False)check_feature_flag = create_feature_flag_dependency(Flag.AUTOMOD)with pytest.raises(HTTPException) as exc_info:@@ -879,7 +880,7 @@ class TestEvaluateFeatureFlag:async def test_a_successful_evaluation_is_authoritative(self, ld_client, user_context):- ld_client.variation.return_value = False+ ld_client.variation_detail.return_value = served(False)assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (False, True)@pytest.mark.asyncio@@ -891,7 +892,7 @@ class TestEvaluateFeatureFlag:async def test_an_evaluation_that_raises_is_not(self, ld_client, user_context):"""The regression this class exists for: a LIVE client can still failto produce a value, and that must not read as a real "off"."""- ld_client.variation.side_effect = Exception("evaluation exploded")+ ld_client.variation_detail.side_effect = Exception("evaluation exploded")assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (False, False)@pytest.mark.asyncio@@ -905,16 +906,16 @@ class TestEvaluateFeatureFlag:"backend.data.user.get_auth_user_flag_fields",new=mocker.AsyncMock(side_effect=ConnectionError("database unreachable")),)- ld_client.variation.return_value = False+ ld_client.variation_detail.return_value = served(False)result = await evaluate_feature_flag(Flag.HIRE_EXPERTS, str(uuid.uuid4()))assert result == (False, False)- ld_client.variation.assert_called_once()+ ld_client.variation_detail.assert_called_once()@pytest.mark.asyncioasync def test_a_non_boolean_flag_value_is_not(self, ld_client, user_context):- ld_client.variation.return_value = {"some": "object"}+ ld_client.variation_detail.return_value = served({"some": "object"})assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (False, False)@pytest.mark.asyncio@@ -934,9 +935,9 @@ class TestEvaluateFeatureFlag:self, ld_client, user_context):"""The refactor must not change what existing callers see."""- ld_client.variation.return_value = True+ ld_client.variation_detail.return_value = served(True)assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is True- ld_client.variation.side_effect = Exception("boom")+ ld_client.variation_detail.side_effect = Exception("boom")assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is False@@ -957,13 +958,13 @@ class TestRequestAttributes:"backend.util.feature_flag._fetch_user_context_status",return_value=(cached, True),)- ld_client.variation.return_value = {"version": "v1"}+ ld_client.variation_detail.return_value = served({"version": "v1"})await feature_flag_module.get_feature_flag_value("card-required-trial-offer", "user-1", None, attributes={"country": "IN"})- evaluated = ld_client.variation.call_args[0][1]+ evaluated = ld_client.variation_detail.call_args[0][1]assert evaluated.get("country") == "IN"assert evaluated.get("email_domain") == "agpt.co"assert cached.get("country") is None@@ -978,7 +979,7 @@ class TestRequestAttributes:await feature_flag_module.get_feature_flag_value("f", "user-1", None, attributes={"key": "someone-else", "kind": "org"})- evaluated = ld_client.variation.call_args[0][1]+ evaluated = ld_client.variation_detail.call_args[0][1]assert (evaluated.key, evaluated.kind) == ("user-1", "user")@pytest.mark.asyncio@@ -1020,3 +1021,8 @@ class TestRequestAttributes:finally:client.close()assert (value == {"version": "v1"}) is offered+++def served(value):+ """LaunchDarkly's answer for a flag it has, as its fallthrough serves it."""+ return EvaluationDetail(value, 0, {"kind": "FALLTHROUGH"})diff --git a/autogpt_platform/backend/scripts/feature_flag_sync_dual_test.py b/autogpt_platform/backend/scripts/feature_flag_sync_dual_test.pynew file mode 100644index 0000000000..1d1c09c4d3--- /dev/null+++ b/autogpt_platform/backend/scripts/feature_flag_sync_dual_test.py@@ -0,0 +1,302 @@+"""What the sync writes, PostHog serves exactly as LaunchDarkly does.++Each vendor's real evaluator answers through the backend's dual comparison:+LaunchDarkly from a synthetic export shaped like Dev's, PostHog from the+definitions the sync maps that export to, evaluated in-process with no remote+fallback.+"""++import asyncio+import json+import logging+import uuid+from datetime import datetime, timezone+from typing import Any++import pytest+from ldclient import LDClient+from ldclient.config import Config as LDConfig+from ldclient.integrations import Files+from posthog import Posthog++import backend.data.db_accessors as db_accessors+import backend.util.feature_flag as ff+import backend.util.feature_flag.posthog as ph+from backend.data.user import AuthUserFlagFields+from backend.util.settings import FeatureFlagBackend+from scripts.feature_flag_sync import map_flag, map_segment, resolve_cohort_refs++ENV = "test"+TARGETED = str(uuid.uuid4())+PEOPLE = {+ "employee": AuthUserFlagFields(+ role="admin",+ email="[email protected]",+ created_at=datetime(2025, 3, 1, tzinfo=timezone.utc),+ ),+ "outsider": AuthUserFlagFields(+ role="authenticated",+ email="[email protected]",+ created_at=datetime(2026, 9, 1, tzinfo=timezone.utc),+ ),+ "targeted": AuthUserFlagFields(role="authenticated", email="[email protected]"),+}++[email protected]("who", [*PEOPLE, "anonymous"])+async def test_every_ported_flag_serves_launchdarklys_answer(+ who, tmp_path, monkeypatch, caplog+):+ ld = _launchdarkly(tmp_path)+ posthog_client = _posthog()+ monkeypatch.setattr(+ ff.settings.config, "feature_flag_backend", FeatureFlagBackend.DUAL+ )+ monkeypatch.setattr(ff, "get_client", lambda: ld)+ monkeypatch.setattr(ph, "get_flag_client", lambda: posthog_client)+ monkeypatch.setattr(db_accessors, "user_db", _AuthTable)+ ff._unanswered_flags_reported.clear()+ posthog_reads = _record_posthog_reads(monkeypatch)+ user_id = "system" if who == "anonymous" else USER_IDS[who]+ # A flag targeting a person cannot be decided in-process without one, so+ # PostHog would answer it remotely; the backend reads only these anonymously.+ keys = ANONYMOUS_READS if who == "anonymous" else [f["key"] for f in EXPORT]++ served: dict[str, Any] = {}+ with caplog.at_level(logging.WARNING, logger=ff.mismatch_logger.name):+ try:+ for key in keys:+ served[key] = await ff.get_feature_flag_value(key, user_id, None)+ while ff._shadow_evaluations:+ await asyncio.gather(*list(ff._shadow_evaluations))+ finally:+ ld.close()+ posthog_client.shutdown()++ assert served == {key: ANSWERS[key].get(who, ANSWERS[key]["*"]) for key in keys}+ assert posthog_reads == {key: (value, True) for key, value in served.items()}+ assert not [+ r.getMessage() for r in caplog.records if r.name == ff.mismatch_logger.name+ ]+++def _segment(key: str, rules: list[list[dict[str, Any]]], **members: Any):+ return {+ "key": key,+ "name": key.title(),+ "rules": [{"clauses": clauses} for clauses in rules],+ **members,+ }+++def _clause(attribute: str, op: str, *values: Any) -> dict[str, Any]:+ # Without a context kind LaunchDarkly reads "/custom/role" as a literal name.+ return {+ "attribute": attribute,+ "op": op,+ "values": list(values),+ "negate": False,+ "contextKind": "user",+ }+++def _in_segment(*keys: str) -> dict[str, Any]:+ return _clause("segmentMatch", "segmentMatch", *keys)+++def _flag(+ key: str,+ variations: list[Any],+ *,+ on: bool = True,+ off: int = 1,+ targets: list[dict[str, Any]] | None = None,+ rules: list[tuple[list[dict[str, Any]], int]] | None = None,+ fallthrough: int = 1,+ names: list[str] | None = None,+) -> dict[str, Any]:+ """A flag as LaunchDarkly's REST API exports it."""+ return {+ "key": key,+ "name": key,+ "kind": "boolean" if variations == [True, False] else "multivariate",+ "variations": [+ {"value": v, **({"name": names[i]} if names else {})}+ for i, v in enumerate(variations)+ ],+ "environments": {+ ENV: {+ "on": on,+ "offVariation": off,+ "targets": targets or [],+ "contextTargets": [],+ "rules": [+ {"clauses": clauses, "variation": variation}+ for clauses, variation in rules or []+ ],+ "fallthrough": {"variation": fallthrough},+ "prerequisites": [],+ }+ },+ }+++USER_IDS = {"employee": str(uuid.uuid4()), "outsider": str(uuid.uuid4())}+USER_IDS["targeted"] = TARGETED++SEGMENTS = [+ _segment(+ "employee", [[_clause("email", "endsWith", "@agpt.co")]], included=[TARGETED]+ ),+ _segment("admin", [[_clause("/custom/role", "in", "admin")]]),+]++LIMITS = {"daily": 625000, "weekly": 3125000}+MULTIPLIERS = {"MAX": 42.66, "PRO": 5}+EXPORT = [+ _flag("AutoMod", [True, False], on=False),+ _flag("artifacts-page", [True, False], fallthrough=0),+ _flag("copilot-voice-mode", [True, False], rules=[([_in_segment("employee")], 0)]),+ _flag(+ "chat-mode-option",+ [True, False],+ rules=[+ ([_clause("email", "in", "[email protected]")], 0),+ ([_in_segment("admin")], 0),+ ],+ ),+ _flag(+ "ai-agent-execution-summary",+ [True, False],+ targets=[{"values": [TARGETED], "variation": 0}],+ rules=[([_clause("email_domain", "in", "agpt.co")], 0)],+ ),+ _flag("copilot-cost-limits", [LIMITS, {}], fallthrough=0),+ _flag(+ "copilot-tier-multipliers",+ [MULTIPLIERS, {}],+ rules=[([_in_segment("employee")], 1)],+ fallthrough=0,+ ),+ _flag("copilot-model-routing", [{"fast": {"standard": "a/b"}}, {}], on=False),+ _flag(+ "copilot-tier-stripe-prices",+ [{"PRO": "price_dev"}, {"PRO": "price_prod"}],+ names=["dev", "prod"],+ fallthrough=0,+ ),+ _flag("stripe-product-id-topup", ["", "prod_topup"], on=False),+]+ANONYMOUS_READS = [+ "AutoMod",+ "artifacts-page",+ "copilot-cost-limits",+ "copilot-tier-multipliers",+ "copilot-model-routing",+ "copilot-tier-stripe-prices",+ "stripe-product-id-topup",+]+# What LaunchDarkly serves each reader, so a fixture that served everyone the+# same could not pass for a port that ignores targeting.+ANSWERS: dict[str, dict[str, Any]] = {+ "AutoMod": {"*": False},+ "artifacts-page": {"*": True},+ "copilot-voice-mode": {"*": False, "employee": True, "targeted": True},+ "chat-mode-option": {"*": False, "employee": True, "outsider": True},+ "ai-agent-execution-summary": {"*": False, "employee": True, "targeted": True},+ "copilot-cost-limits": {"*": LIMITS},+ "copilot-tier-multipliers": {"*": MULTIPLIERS, "employee": {}, "targeted": {}},+ "copilot-model-routing": {"*": {}},+ "copilot-tier-stripe-prices": {"*": {"PRO": "price_dev"}},+ "stripe-product-id-topup": {"*": "prod_topup"},+}+++def _launchdarkly(tmp_path) -> LDClient:+ """LaunchDarkly's own evaluator over the export, in its SDK data format."""+ path = tmp_path / "launchdarkly.json"+ path.write_text(+ json.dumps(+ {+ "flags": {f["key"]: _sdk_flag(f) for f in EXPORT},+ "segments": {s["key"]: _sdk_segment(s) for s in SEGMENTS},+ }+ )+ )+ client = LDClient(+ LDConfig(+ "sdk-test",+ update_processor_class=Files.new_data_source(paths=[str(path)]),+ send_events=False,+ )+ )+ assert client.is_initialized()+ return client+++def _sdk_flag(flag: dict[str, Any]) -> dict[str, Any]:+ cfg = flag["environments"][ENV]+ return {+ **{k: cfg[k] for k in ("on", "offVariation", "targets", "contextTargets")},+ "key": flag["key"],+ "version": 1,+ "salt": flag["key"],+ "variations": [v["value"] for v in flag["variations"]],+ "rules": [{"id": f"rule-{i}", **r} for i, r in enumerate(cfg["rules"])],+ "fallthrough": cfg["fallthrough"],+ "prerequisites": [],+ }+++def _sdk_segment(segment: dict[str, Any]) -> dict[str, Any]:+ return {+ "key": segment["key"],+ "version": 1,+ "salt": segment["key"],+ "included": segment.get("included", []),+ "excluded": [],+ "rules": [{"id": f"rule-{i}", **r} for i, r in enumerate(segment["rules"])],+ }+++def _posthog() -> Posthog:+ """PostHog's in-process evaluator over what the sync would write."""+ cohorts = {s["key"]: map_segment(s, ENV) for s in SEGMENTS}+ payloads = [c.payload for c in cohorts.values() if c.payload is not None]+ ids = {p["name"]: i for i, p in enumerate(payloads, start=1)}+ flags = []+ for i, exported in enumerate(EXPORT, start=1):+ mapped = map_flag(exported, ENV, cohorts)+ assert mapped.payload is not None, mapped.decisions+ filters = resolve_cohort_refs(mapped.payload["filters"], ids)+ flags.append({**mapped.payload, "id": i, "filters": filters})++ client = Posthog("phc-test", secret_key="phx-test", enable_local_evaluation=False)+ client.feature_flags = flags+ client.cohorts = {str(ids[p["name"]]): p["filters"]["properties"] for p in payloads}+ client._get_flags_decision = _no_remote_flags+ return client+++def _no_remote_flags(*args, **kwargs):+ raise ConnectionError("only an in-process answer counts here")+++def _record_posthog_reads(monkeypatch) -> dict[str, tuple[Any, bool]]:+ reads: dict[str, tuple[Any, bool]] = {}+ evaluate = ph.evaluate_flag++ async def recording(flag_key: str, *args: Any, **kwargs: Any):+ reads[flag_key] = await evaluate(flag_key, *args, **kwargs)+ return reads[flag_key]++ monkeypatch.setattr(ph, "evaluate_flag", recording)+ return reads+++class _AuthTable:+ """The auth rows the backend builds both vendors' user context from."""++ async def get_auth_user_flag_fields(self, user_id: str):+ return {USER_IDS[who]: fields for who, fields in PEOPLE.items()}[user_id]diff --git a/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsx b/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsxindex 2e0eb027c6..67599d5549 100644--- a/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsx+++ b/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsx@@ -71,6 +71,7 @@ describe("PostHogUserTracker", () => {render(<PostHogUserTracker />);expect(posthog.setPersonPropertiesForFlags).toHaveBeenCalledWith({+ email: "[email protected]",email_domain: "example.com",role: "authenticated",created_at: "2026-05-08T12:00:00Z",diff --git a/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.tsindex 927d4e7c23..898f360dc2 100644--- a/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.ts+++ b/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.ts@@ -103,8 +103,9 @@ describe("buildLDContext", () => {});describe("buildFlagPersonProperties", () => {- it("carries the attributes the ported rules target on, never the raw email", () => {+ it("carries the attributes the ported rules target on", () => {expect(buildFlagPersonProperties(userFixture())).toEqual({+ email: "[email protected]",email_domain: "example.com",role: "authenticated",created_at: "2026-05-08T12:00:00Z",@@ -116,6 +117,9 @@ describe("buildFlagPersonProperties", () => {userFixture({ role: undefined, created_at: undefined }),);- expect(properties).toEqual({ email_domain: "example.com" });+ expect(properties).toEqual({+ email: "[email protected]",+ email_domain: "example.com",+ });});});diff --git a/autogpt_platform/frontend/src/services/feature-flags/helpers.ts b/autogpt_platform/frontend/src/services/feature-flags/helpers.tsindex 15d4dea0dd..1febab8e25 100644--- a/autogpt_platform/frontend/src/services/feature-flags/helpers.ts+++ b/autogpt_platform/frontend/src/services/feature-flags/helpers.ts@@ -32,11 +32,11 @@ export interface LDMultiContext {export type LDContext = LDUserContext | LDMultiContext;// The attributes the ported flag targeting rules match on, in the flat shape-// PostHog person properties take. No raw email: no rule reads it, and-// individual targets key on the distinct id.+// PostHog person properties take; the synced employee cohort matches on email.export function buildFlagPersonProperties(user: User): Record<string, string> {return {...(user.email && {+ email: user.email,email_domain: user.email.split("@").at(-1) ?? "",}),...(user.role && { role: user.role }),