Skip to content

AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.

fix(platform): PostHog flag reads carry email, and dual mode logs only real disagreements (#15357)

Reinier van der Leer4473fbbcb195
diff --git a/autogpt_platform/backend/backend/util/feature_flag/__init__.py b/autogpt_platform/backend/backend/util/feature_flag/__init__.py
index 18936f41a8..3c49d323c0 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/__init__.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/__init__.py
@@ -15,6 +15,7 @@ from autogpt_libs.auth.dependencies import get_optional_user_id
from fastapi import HTTPException, Security
from ldclient import Context, LDClient
from ldclient.config import Config
+from ldclient.evaluation import EvaluationDetail
from typing_extensions import ParamSpec
from backend.util.cache import cached
@@ -40,8 +41,8 @@ _init_attempted = False
_shadow_evaluations: set[asyncio.Task] = set()
_shadow_evaluations_stopped = False
MAX_CONCURRENT_SHADOW_EVALUATIONS = 100
-# Flags PostHog could not answer for, already reported once. Before phase 2
-# creates the flags that is every read; one record per flag keeps the signal.
+# Flags PostHog could not answer while LaunchDarkly could, already reported
+# once: such a flag fails every read, so one record per flag keeps the signal.
_unanswered_flags_reported: set[str] = set()
@@ -210,38 +211,6 @@ class Flag(str, Enum):
# a cohort before it reaches everyone.
CHAT_CONNECTION_UPSELL = "chat-connection-upsell"
- # Shrinks what Otto reads: strips builder-UI annotations from the
- # block schemas, and digests oversized tool results to the workspace.
-
-
-# LaunchDarkly keys whose targeting is segment membership or an individual-user
-# target — neither of which any PostHog cohort reproduces until phase 2 creates
-# one. From ld-targeting-summary.md; attribute rules (email domain, role,
-# signup date) are not listed because `_person_properties` already carries them.
-LD_UNPORTED_TARGETING = frozenset(
- {
- "ai-agent-execution-summary",
- "artifacts",
- "artifacts-page",
- "autogpt-new-layout",
- "AutoMod",
- "beta-blocks",
- "chat",
- "chat-mode-option",
- "chat-search",
- "chat-sharing",
- "copilot-bot-platforms",
- "dream-pass-enabled",
- "enable-platform-payment",
- "generic-trigger-agents",
- "graphiti-memory",
- "new-tool-ui",
- "nightly-copilot",
- "SHOW_ORG_SETTINGS",
- "task-progress-bar",
- }
-)
-
def initialize_feature_flags() -> None:
"""Start whichever vendor(s) the configured backend reads from."""
@@ -614,11 +583,12 @@ async def _record_mismatch(
logger.warning(f"PostHog shadow evaluation raised for {flag_key}: {e}")
return
- if ld_result == ph_result:
- return
-
ld_value, ld_evaluated = ld_result
ph_value, ph_evaluated = ph_result
+ # A mismatch is a read whose served value would change at the switch, or
+ # one PostHog cannot answer where LaunchDarkly can.
+ if ph_value == ld_value and (ph_evaluated or not ld_evaluated):
+ return
if not ph_evaluated:
if flag_key in _unanswered_flags_reported:
return
@@ -631,11 +601,6 @@ async def _record_mismatch(
"user": _user_digest(user_id),
"launchdarkly": {"value": ld_value, "evaluated": ld_evaluated},
"posthog": {"value": ph_value, "evaluated": ph_evaluated},
- # Segment and individual-user targeting has no PostHog cohort
- # until phase 2 builds one, so these flags are expected to
- # disagree — the diff-week report has to separate them from
- # real divergence rather than drown in them.
- "expected_until_cohorts_exist": flag_key in LD_UNPORTED_TARGETING,
},
default=repr,
sort_keys=True,
@@ -677,9 +642,10 @@ async def _evaluate_launchdarkly(
``evaluated`` is False whenever *default* is standing in for an answer
LaunchDarkly could not give — no client, an uninitialised one, a failed
- user-context lookup, or an evaluation that raised. An initialised client
- is not on its own enough: the context lookup is a database read, so a
- live client can still fail to produce a value.
+ user-context lookup, a flag LaunchDarkly does not have (or has switched
+ off with no off variation), or an evaluation that raised. An initialised
+ client is not on its own enough: the context lookup is a database read,
+ so a live client can still fail to produce a value.
"""
try:
client = get_client()
@@ -696,15 +662,18 @@ async def _evaluate_launchdarkly(
user_id
)
- # Evaluate flag
- result = client.variation(flag_key, user_context, default)
+ detail = client.variation_detail(flag_key, user_context, default)
logger.debug(
- f"Feature flag {flag_key} for user {user_id}: {result} (type: {type(result).__name__})"
+ "Feature flag %s for user %s: %r (%s)",
+ flag_key,
+ user_id,
+ detail.value,
+ detail.reason,
)
# A degraded context evaluates fine, it just answers for an anonymous
# user rather than this one — so the value is a guess, not an answer.
- return result, context_resolved
+ return detail.value, context_resolved and not detail.is_default_value()
except Exception as e:
logger.warning(
@@ -718,14 +687,15 @@ def _person_properties(context: Context) -> dict[str, Any]:
Reads the LaunchDarkly context rather than the auth row so both vendors
see one cached lookup and cannot drift; ``custom.role`` is dropped
- because PostHog targets flat properties, and ``email`` because no ported
- rule reads it — individual targets key on the ``distinct_id``.
+ because PostHog targets flat properties. ``email`` is sent because the
+ ported email rules and the ``ld:employee`` cohort match on it, and a rule
+ on a property the read omits cannot be evaluated in-process.
"""
if context.anonymous:
return {}
return {
attribute: context.get(attribute)
- for attribute in ("role", "email_domain", "created_at", "country")
+ for attribute in ("role", "email", "email_domain", "created_at", "country")
if context.get(attribute) is not None
}
@@ -1072,11 +1042,14 @@ def _flag_backend_initialized() -> bool:
@contextlib.contextmanager
def mock_flag_variation(flag_key: str, return_value: Any):
"""Context manager for testing feature flags."""
- original_variation = get_client().variation
- get_client().variation = lambda key, context, default: (
- return_value if key == flag_key else original_variation(key, context, default)
+ client = get_client()
+ original = client.variation_detail
+ client.variation_detail = lambda key, context, default: (
+ EvaluationDetail(return_value, 0, {"kind": "FALLTHROUGH"})
+ if key == flag_key
+ else original(key, context, default)
)
try:
yield
finally:
- get_client().variation = original_variation
+ client.variation_detail = original
diff --git a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py b/autogpt_platform/backend/backend/util/feature_flag/backends_test.py
index 77eaf1857c..c8e11bc15d 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/backends_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/backends_test.py
@@ -12,6 +12,7 @@ import sentry_sdk.feature_flags
from fastapi import HTTPException
from ldclient import Context, LDClient
from ldclient.config import Config as LDConfig
+from ldclient.evaluation import EvaluationDetail
from ldclient.integrations.test_data import TestData
from posthog import Posthog
from sentry_sdk.tracing import Span
@@ -114,17 +115,17 @@ class TestDefaultBackendIsUnchanged:
self, mocker, ld_client, user_context
):
posthog = mocker.patch.object(ph, "evaluate_flag")
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)
- ld_client.variation.assert_called_once()
+ ld_client.variation_detail.assert_called_once()
posthog.assert_not_called()
@pytest.mark.asyncio
async def test_the_default_reads_launchdarkly_for_raw_values(
self, ld_client, user_context
):
- ld_client.variation.return_value = {"daily": 5}
+ ld_client.variation_detail.return_value = served({"daily": 5})
assert await ff.get_feature_flag_value("copilot-cost-limits", "system") == {
"daily": 5
@@ -231,7 +232,7 @@ class TestPostHogBackend:
async def test_targeting_attributes_are_passed_as_person_properties(self, mocker):
"""Every attribute an LD rule targets on has to reach PostHog, or the
flag silently evaluates against a user without them — and nothing
- more: the raw email is not one of them."""
+ more: ``custom`` is LaunchDarkly's nesting of ``role``."""
use_backend(mocker, FeatureFlagBackend.POSTHOG)
client = stub_posthog(mocker, value=True)
context = (
@@ -255,10 +256,40 @@ class TestPostHogBackend:
_, kwargs = client.evaluate_flags.call_args
assert kwargs["person_properties"] == {
"role": "admin",
+ "email": "[email protected]",
"email_domain": "agpt.co",
"created_at": "2026-05-07T12:00:00+00:00",
}
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "email,expected", [("[email protected]", (True, True)), ("[email protected]", (False, True))]
+ )
+ async def test_an_email_cohort_evaluates_in_process(self, mocker, email, expected):
+ """The synced ``ld:employee`` cohort matches on ``email``; without it a
+ read cannot be decided locally and becomes a remote /flags call."""
+ use_backend(mocker, FeatureFlagBackend.POSTHOG)
+ posthog_client = _posthog_with_an_employee_cohort()
+ mocker.patch.object(ph, "get_flag_client", return_value=posthog_client)
+ context = (
+ Context.builder("u-1")
+ .kind("user")
+ .anonymous(False)
+ .set("email", email)
+ .set("email_domain", email.split("@")[1])
+ .build()
+ )
+ mocker.patch(
+ "backend.util.feature_flag._fetch_user_context_status",
+ return_value=(context, True),
+ )
+ try:
+ result = await evaluate_feature_flag(Flag.COPILOT_VOICE_MODE, "u-1")
+ finally:
+ posthog_client.shutdown()
+
+ assert result == expected
+
@pytest.mark.asyncio
async def test_an_anonymous_context_carries_no_person_properties(self, mocker):
"""The `"system"`-keyed config flags have no user to describe."""
@@ -296,7 +327,7 @@ class TestDualBackend:
self, mocker, ld_client, user_context
):
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
stub_posthog(mocker, value=False)
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)
@@ -306,7 +337,7 @@ class TestDualBackend:
self, mocker, ld_client, user_context, caplog
):
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
stub_posthog(mocker, value=False)
user_id = str(uuid.uuid4())
@@ -333,7 +364,7 @@ class TestDualBackend:
self, mocker, ld_client, user_context, caplog
):
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
stub_posthog(mocker, value=True)
with caplog.at_level(
@@ -353,7 +384,7 @@ class TestDualBackend:
"""Both say "off", but only one of them knows it — that is the
difference the diff week exists to find."""
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = False
+ ld_client.variation_detail.return_value = served(False)
stub_posthog(mocker, value=None)
with caplog.at_level(
@@ -380,7 +411,7 @@ class TestDualBackend:
"""Before phase 2 creates the flags PostHog answers nothing, so every
read would otherwise log the same record."""
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
stub_posthog(mocker, value=None)
with caplog.at_level(
@@ -405,7 +436,7 @@ class TestDualBackend:
self, mocker, ld_client, user_context, caplog
):
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
stub_posthog(mocker, value=False)
with caplog.at_level(
@@ -426,12 +457,40 @@ class TestDualBackend:
== 3
)
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "posthog_value", [None, True], ids=["posthog-lacks-it", "posthog-serves-it"]
+ )
+ async def test_a_flag_launchdarkly_lacks_logs_nothing_if_the_value_holds(
+ self, mocker, user_context, caplog, posthog_value
+ ):
+ """LaunchDarkly serves the caller's default; a PostHog read serving the
+ same changes nothing at the switch, answered or not."""
+ use_backend(mocker, FeatureFlagBackend.DUAL)
+ ld = _launchdarkly_with(TestData.data_source())
+ mocker.patch("backend.util.feature_flag.ldclient.get", return_value=ld)
+ stub_posthog(mocker, value=posthog_value)
+
+ with caplog.at_level(
+ logging.WARNING, logger="backend.util.feature_flag.mismatch"
+ ):
+ try:
+ result = await evaluate_feature_flag(Flag.COPILOT_SDK, "u-1", True)
+ await drain_shadow_evaluations()
+ finally:
+ ld.close()
+
+ assert result == (True, False)
+ assert not [
+ r for r in caplog.records if r.name == "backend.util.feature_flag.mismatch"
+ ]
+
@pytest.mark.asyncio
async def test_a_posthog_failure_cannot_break_the_read(
self, mocker, ld_client, user_context
):
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
mocker.patch.object(ph, "evaluate_flag", side_effect=Exception("boom"))
assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is True
@@ -443,7 +502,7 @@ class TestDualBackend:
):
"""A mismatch record must never be the thing that raises."""
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = object()
+ ld_client.variation_detail.return_value = served(object())
stub_posthog(mocker, value=True)
with caplog.at_level(
@@ -474,6 +533,39 @@ class TestDualBackend:
assert ff._flag_backend_initialized() is False
+class TestLaunchDarklyAnswers:
+ """Through LaunchDarkly's real evaluator: only a served variation is an answer."""
+
+ @pytest.mark.asyncio
+ @pytest.mark.parametrize(
+ "configure,expected",
+ [
+ (lambda td: td.update(td.flag("AutoMod").boolean_flag()), (True, True)),
+ (
+ lambda td: td.update(td.flag("AutoMod").boolean_flag().on(False)),
+ (False, True),
+ ),
+ (lambda td: None, (True, False)),
+ ],
+ ids=["on", "switched-off", "absent"],
+ )
+ async def test_only_a_served_variation_is_authoritative(
+ self, mocker, user_context, configure, expected
+ ):
+ """A switched-off flag still answers with its off variation; a flag
+ LaunchDarkly does not have answers nothing, whatever the default."""
+ td = TestData.data_source()
+ configure(td)
+ ld = _launchdarkly_with(td)
+ mocker.patch("backend.util.feature_flag.ldclient.get", return_value=ld)
+ try:
+ result = await evaluate_feature_flag(Flag.AUTOMOD, "u-1", True)
+ finally:
+ ld.close()
+
+ assert result == expected
+
+
class TestDualStaysOffTheRequestPath:
"""The shadow answer is discarded, so nobody may wait for it."""
@@ -482,7 +574,7 @@ class TestDualStaysOffTheRequestPath:
self, mocker, ld_client, user_context
):
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
posthog_started = asyncio.Event()
async def slow_posthog(*args, **kwargs):
@@ -506,7 +598,7 @@ class TestDualStaysOffTheRequestPath:
"""The failed-lookup path is deliberately uncached, so evaluating the
two vendors independently would double its database reads."""
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
stub_posthog(mocker, value=True)
lookup = mocker.patch(
"backend.util.feature_flag._fetch_user_context_status",
@@ -523,7 +615,7 @@ class TestDualStaysOffTheRequestPath:
"""A slow PostHog costs the diff week its samples, not the process
its memory."""
use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
mocker.patch.object(ff, "MAX_CONCURRENT_SHADOW_EVALUATIONS", 2)
async def never_finishes(*args, **kwargs):
@@ -539,29 +631,6 @@ class TestDualStaysOffTheRequestPath:
for task in list(ff._shadow_evaluations):
task.cancel()
- @pytest.mark.asyncio
- async def test_segment_targeted_flags_are_marked_as_expected(
- self, mocker, ld_client, user_context, caplog
- ):
- """Until phase 2 builds the cohorts these disagree by construction, and
- the diff-week report has to be able to set them aside."""
- use_backend(mocker, FeatureFlagBackend.DUAL)
- ld_client.variation.return_value = True
- stub_posthog(mocker, value=False)
-
- with caplog.at_level(
- logging.WARNING, logger="backend.util.feature_flag.mismatch"
- ):
- await evaluate_feature_flag(Flag.GRAPHITI_MEMORY, "u-1")
- await drain_shadow_evaluations()
-
- [message] = [
- r.getMessage()
- for r in caplog.records
- if r.name == "backend.util.feature_flag.mismatch"
- ]
- assert _mismatch_record(message)["expected_until_cohorts_exist"] is True
-
class TestShutdownStopsShadowEvaluations:
"""A shadow read that outlives teardown rebuilds the client it just closed:
@@ -581,7 +650,7 @@ class TestShutdownStopsShadowEvaluations:
use_backend(mocker, FeatureFlagBackend.DUAL)
mocker.patch.object(ff, "shutdown_launchdarkly")
mocker.patch.object(ph, "shutdown_posthog_flags")
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
async def never_finishes(*args, **kwargs):
await asyncio.sleep(30)
@@ -602,7 +671,7 @@ class TestShutdownStopsShadowEvaluations:
use_backend(mocker, FeatureFlagBackend.DUAL)
mocker.patch.object(ff, "shutdown_launchdarkly")
mocker.patch.object(ph, "shutdown_posthog_flags")
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
get_client = mocker.patch.object(ph, "get_flag_client")
ff.shutdown_feature_flags()
@@ -710,7 +779,7 @@ class TestSentryFlagContext:
self, mocker, ld_client, user_context, sentry_flags, backend
):
use_backend(mocker, backend)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
stub_posthog(mocker, value=backend is FeatureFlagBackend.POSTHOG)
assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is True
@@ -722,7 +791,7 @@ class TestSentryFlagContext:
self, ld_client, user_context, sentry_flags, initialized
):
ld_client.is_initialized.return_value = initialized
- ld_client.variation.return_value = {"daily": 5}
+ ld_client.variation_detail.return_value = served({"daily": 5})
await ff.get_feature_flag_value("copilot-cost-limits", "u-1", {"daily": 1})
assert sentry_flags.get() == []
@@ -746,7 +815,7 @@ class TestSentryFlagContext:
async def test_a_non_boolean_answer_to_a_boolean_read_records_the_default(
self, ld_client, user_context, sentry_flags
):
- ld_client.variation.return_value = "on"
+ ld_client.variation_detail.return_value = served("on")
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1", True) == (
True,
@@ -781,7 +850,7 @@ class TestSentryFlagContext:
use_backend(mocker, FeatureFlagBackend.LAUNCHDARKLY)
mocker.patch("backend.util.feature_flag.is_configured", return_value=configured)
ld_client.is_initialized.return_value = initialized
- ld_client.variation.return_value = answer
+ ld_client.variation_detail.return_value = served(answer)
# A stale earlier answer must not be what an error from this route carries.
sentry_sdk.feature_flags.add_feature_flag(Flag.HIRE_EXPERTS.value, True)
@@ -820,7 +889,7 @@ class TestSentryFlagContext:
sentry_sdk.feature_flags.add_feature_flag(
f"{Flag.HIRE_EXPERTS.value}.fallback", True
)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1")
@@ -838,7 +907,7 @@ class TestSentryFlagContext:
"add_feature_flag",
side_effect=RuntimeError("sentry down"),
)
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)
@@ -889,6 +958,10 @@ def _launchdarkly_serving_all_but_india() -> LDClient:
.if_not_match("country", "IN")
.then_return(1)
)
+ return _launchdarkly_with(td)
+
+
+def _launchdarkly_with(td: TestData) -> LDClient:
return LDClient(LDConfig("sdk-test", update_processor_class=td, send_events=False))
@@ -929,6 +1002,46 @@ def _posthog_serving_all_but_india() -> Posthog:
return client
+def _posthog_with_an_employee_cohort() -> Posthog:
+ """The shape the sync gives an email-targeted segment, evaluated locally."""
+ client = Posthog("phc-test", secret_key="phx-test", enable_local_evaluation=False)
+ client.feature_flags = [
+ {
+ "id": 1,
+ "key": Flag.COPILOT_VOICE_MODE.value,
+ "active": True,
+ "filters": {
+ "groups": [
+ {
+ "properties": [{"key": "id", "type": "cohort", "value": 7}],
+ "rollout_percentage": 100,
+ }
+ ]
+ },
+ }
+ ]
+ client.cohorts = {
+ "7": {
+ "type": "OR",
+ "values": [
+ {
+ "type": "AND",
+ "values": [
+ {
+ "key": "email",
+ "type": "person",
+ "operator": "regex",
+ "value": r"@agpt\.co$",
+ }
+ ],
+ }
+ ],
+ }
+ }
+ client._get_flags_decision = _no_remote_flags
+ return client
+
+
def _no_remote_flags(*args, **kwargs):
raise ConnectionError("remote /flags is unavailable in tests")
@@ -941,3 +1054,8 @@ def _gated_route():
return "served"
return route
+
+
+def served(value):
+ """LaunchDarkly's answer for a flag it has, as its fallthrough serves it."""
+ return EvaluationDetail(value, 0, {"kind": "FALLTHROUGH"})
diff --git a/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py b/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py
index cc6381f3ff..365aa1b1e1 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/definition_cache_test.py
@@ -7,6 +7,7 @@ import uuid
import pytest
from ldclient import Context, LDClient
+from ldclient.evaluation import EvaluationDetail
from posthog import Posthog
from posthog.request import GetResponse
from pydantic import ValidationError
@@ -418,7 +419,7 @@ class TestDefaultBackendIsUntouched:
self, mocker, ld_client, user_context
):
build = mocker.patch.object(ph, "get_flag_definition_cache")
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (True, True)
build.assert_not_called()
@@ -562,3 +563,8 @@ class TestMetrics:
provider(redis).get_flag_definitions()
assert REGISTRY.get_sample_value(name, {"outcome": "cached"}) == before + 1
+
+
+def served(value):
+ """LaunchDarkly's answer for a flag it has, as its fallthrough serves it."""
+ return EvaluationDetail(value, 0, {"kind": "FALLTHROUGH"})
diff --git a/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.py b/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.py
index 3b57d14d79..66222233f1 100644
--- a/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.py
+++ b/autogpt_platform/backend/backend/util/feature_flag/feature_flag_test.py
@@ -8,6 +8,7 @@ import pytest
from fastapi import HTTPException
from ldclient import Context, LDClient
from ldclient.config import Config
+from ldclient.evaluation import EvaluationDetail
from ldclient.integrations.test_data import TestData
import backend
@@ -39,7 +40,7 @@ def ld_client(mocker):
@pytest.mark.asyncio
async def test_feature_flag_enabled(ld_client):
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
@feature_flag("test-flag")
async def test_function(user_id: str):
@@ -47,12 +48,12 @@ async def test_feature_flag_enabled(ld_client):
result = await test_function(user_id="test-user")
assert result == "success"
- ld_client.variation.assert_called_once()
+ ld_client.variation_detail.assert_called_once()
@pytest.mark.asyncio
async def test_feature_flag_unauthorized_response(ld_client):
- ld_client.variation.return_value = False
+ ld_client.variation_detail.return_value = served(False)
@feature_flag("test-flag")
async def test_function(user_id: str):
@@ -65,23 +66,23 @@ async def test_feature_flag_unauthorized_response(ld_client):
def test_mock_flag_variation(ld_client):
with mock_flag_variation("test-flag", True):
- assert ld_client.variation("test-flag", None, False) is True
+ assert ld_client.variation_detail("test-flag", None, False).value is True
with mock_flag_variation("test-flag", False):
- assert ld_client.variation("test-flag", None, True) is False
+ assert ld_client.variation_detail("test-flag", None, True).value is False
@pytest.mark.asyncio
async def test_is_feature_enabled(ld_client):
"""Test the is_feature_enabled helper function."""
ld_client.is_initialized.return_value = True
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
result = await is_feature_enabled(Flag.AUTOMOD, "user123", default=False)
assert result is True
- ld_client.variation.assert_called_once()
- call_args = ld_client.variation.call_args
+ ld_client.variation_detail.assert_called_once()
+ call_args = ld_client.variation_detail.call_args
assert call_args[0][0] == "AutoMod" # flag_key
assert call_args[0][2] is False # default value
@@ -94,7 +95,7 @@ async def test_is_feature_enabled_not_initialized(ld_client):
result = await is_feature_enabled(Flag.AUTOMOD, "user123", default=True)
assert result is True # Should return default
- ld_client.variation.assert_not_called()
+ ld_client.variation_detail.assert_not_called()
@pytest.mark.asyncio
@@ -403,13 +404,13 @@ class TestEnvOverrideWiring:
return "success"
assert await test_function(user_id="test-user") == "success"
- ld_client.variation.assert_not_called()
+ ld_client.variation_detail.assert_not_called()
@pytest.mark.asyncio
async def test_decorator_per_flag_false_wins_over_launchdarkly(
self, ld_client, monkeypatch: pytest.MonkeyPatch
):
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
monkeypatch.setenv("FORCE_FLAG_TEST_FLAG", "false")
@feature_flag("test-flag")
@@ -419,7 +420,7 @@ class TestEnvOverrideWiring:
with pytest.raises(HTTPException) as exc_info:
await test_function(user_id="test-user")
assert exc_info.value.status_code == 404
- ld_client.variation.assert_not_called()
+ ld_client.variation_detail.assert_not_called()
@pytest.mark.asyncio
async def test_dependency_force_all_wins_over_unconfigured_sdk(
@@ -456,7 +457,7 @@ class TestEnvOverrideWiring:
explicit passthrough every request to a disabled route files an
ERROR and buries real flag-evaluation failures.
"""
- ld_client.variation.return_value = False
+ ld_client.variation_detail.return_value = served(False)
@feature_flag("test-flag")
async def test_function(user_id: str):
@@ -496,7 +497,7 @@ class TestEnvOverrideWiring:
"""
mocker.patch("backend.util.feature_flag.is_configured", return_value=True)
ld_client.is_initialized.return_value = True
- ld_client.variation.return_value = False
+ ld_client.variation_detail.return_value = served(False)
check_feature_flag = create_feature_flag_dependency(Flag.AUTOMOD)
with pytest.raises(HTTPException) as exc_info:
@@ -879,7 +880,7 @@ class TestEvaluateFeatureFlag:
async def test_a_successful_evaluation_is_authoritative(
self, ld_client, user_context
):
- ld_client.variation.return_value = False
+ ld_client.variation_detail.return_value = served(False)
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (False, True)
@pytest.mark.asyncio
@@ -891,7 +892,7 @@ class TestEvaluateFeatureFlag:
async def test_an_evaluation_that_raises_is_not(self, ld_client, user_context):
"""The regression this class exists for: a LIVE client can still fail
to produce a value, and that must not read as a real "off"."""
- ld_client.variation.side_effect = Exception("evaluation exploded")
+ ld_client.variation_detail.side_effect = Exception("evaluation exploded")
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (False, False)
@pytest.mark.asyncio
@@ -905,16 +906,16 @@ class TestEvaluateFeatureFlag:
"backend.data.user.get_auth_user_flag_fields",
new=mocker.AsyncMock(side_effect=ConnectionError("database unreachable")),
)
- ld_client.variation.return_value = False
+ ld_client.variation_detail.return_value = served(False)
result = await evaluate_feature_flag(Flag.HIRE_EXPERTS, str(uuid.uuid4()))
assert result == (False, False)
- ld_client.variation.assert_called_once()
+ ld_client.variation_detail.assert_called_once()
@pytest.mark.asyncio
async def test_a_non_boolean_flag_value_is_not(self, ld_client, user_context):
- ld_client.variation.return_value = {"some": "object"}
+ ld_client.variation_detail.return_value = served({"some": "object"})
assert await evaluate_feature_flag(Flag.HIRE_EXPERTS, "u-1") == (False, False)
@pytest.mark.asyncio
@@ -934,9 +935,9 @@ class TestEvaluateFeatureFlag:
self, ld_client, user_context
):
"""The refactor must not change what existing callers see."""
- ld_client.variation.return_value = True
+ ld_client.variation_detail.return_value = served(True)
assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is True
- ld_client.variation.side_effect = Exception("boom")
+ ld_client.variation_detail.side_effect = Exception("boom")
assert await is_feature_enabled(Flag.HIRE_EXPERTS, "u-1") is False
@@ -957,13 +958,13 @@ class TestRequestAttributes:
"backend.util.feature_flag._fetch_user_context_status",
return_value=(cached, True),
)
- ld_client.variation.return_value = {"version": "v1"}
+ ld_client.variation_detail.return_value = served({"version": "v1"})
await feature_flag_module.get_feature_flag_value(
"card-required-trial-offer", "user-1", None, attributes={"country": "IN"}
)
- evaluated = ld_client.variation.call_args[0][1]
+ evaluated = ld_client.variation_detail.call_args[0][1]
assert evaluated.get("country") == "IN"
assert evaluated.get("email_domain") == "agpt.co"
assert cached.get("country") is None
@@ -978,7 +979,7 @@ class TestRequestAttributes:
await feature_flag_module.get_feature_flag_value(
"f", "user-1", None, attributes={"key": "someone-else", "kind": "org"}
)
- evaluated = ld_client.variation.call_args[0][1]
+ evaluated = ld_client.variation_detail.call_args[0][1]
assert (evaluated.key, evaluated.kind) == ("user-1", "user")
@pytest.mark.asyncio
@@ -1020,3 +1021,8 @@ class TestRequestAttributes:
finally:
client.close()
assert (value == {"version": "v1"}) is offered
+
+
+def served(value):
+ """LaunchDarkly's answer for a flag it has, as its fallthrough serves it."""
+ return EvaluationDetail(value, 0, {"kind": "FALLTHROUGH"})
diff --git a/autogpt_platform/backend/scripts/feature_flag_sync_dual_test.py b/autogpt_platform/backend/scripts/feature_flag_sync_dual_test.py
new file mode 100644
index 0000000000..1d1c09c4d3
--- /dev/null
+++ b/autogpt_platform/backend/scripts/feature_flag_sync_dual_test.py
@@ -0,0 +1,302 @@
+"""What the sync writes, PostHog serves exactly as LaunchDarkly does.
+
+Each vendor's real evaluator answers through the backend's dual comparison:
+LaunchDarkly from a synthetic export shaped like Dev's, PostHog from the
+definitions the sync maps that export to, evaluated in-process with no remote
+fallback.
+"""
+
+import asyncio
+import json
+import logging
+import uuid
+from datetime import datetime, timezone
+from typing import Any
+
+import pytest
+from ldclient import LDClient
+from ldclient.config import Config as LDConfig
+from ldclient.integrations import Files
+from posthog import Posthog
+
+import backend.data.db_accessors as db_accessors
+import backend.util.feature_flag as ff
+import backend.util.feature_flag.posthog as ph
+from backend.data.user import AuthUserFlagFields
+from backend.util.settings import FeatureFlagBackend
+from scripts.feature_flag_sync import map_flag, map_segment, resolve_cohort_refs
+
+ENV = "test"
+TARGETED = str(uuid.uuid4())
+PEOPLE = {
+ "employee": AuthUserFlagFields(
+ role="admin",
+ created_at=datetime(2025, 3, 1, tzinfo=timezone.utc),
+ ),
+ "outsider": AuthUserFlagFields(
+ role="authenticated",
+ created_at=datetime(2026, 9, 1, tzinfo=timezone.utc),
+ ),
+ "targeted": AuthUserFlagFields(role="authenticated", email="[email protected]"),
+}
+
+
[email protected]("who", [*PEOPLE, "anonymous"])
+async def test_every_ported_flag_serves_launchdarklys_answer(
+ who, tmp_path, monkeypatch, caplog
+):
+ ld = _launchdarkly(tmp_path)
+ posthog_client = _posthog()
+ monkeypatch.setattr(
+ ff.settings.config, "feature_flag_backend", FeatureFlagBackend.DUAL
+ )
+ monkeypatch.setattr(ff, "get_client", lambda: ld)
+ monkeypatch.setattr(ph, "get_flag_client", lambda: posthog_client)
+ monkeypatch.setattr(db_accessors, "user_db", _AuthTable)
+ ff._unanswered_flags_reported.clear()
+ posthog_reads = _record_posthog_reads(monkeypatch)
+ user_id = "system" if who == "anonymous" else USER_IDS[who]
+ # A flag targeting a person cannot be decided in-process without one, so
+ # PostHog would answer it remotely; the backend reads only these anonymously.
+ keys = ANONYMOUS_READS if who == "anonymous" else [f["key"] for f in EXPORT]
+
+ served: dict[str, Any] = {}
+ with caplog.at_level(logging.WARNING, logger=ff.mismatch_logger.name):
+ try:
+ for key in keys:
+ served[key] = await ff.get_feature_flag_value(key, user_id, None)
+ while ff._shadow_evaluations:
+ await asyncio.gather(*list(ff._shadow_evaluations))
+ finally:
+ ld.close()
+ posthog_client.shutdown()
+
+ assert served == {key: ANSWERS[key].get(who, ANSWERS[key]["*"]) for key in keys}
+ assert posthog_reads == {key: (value, True) for key, value in served.items()}
+ assert not [
+ r.getMessage() for r in caplog.records if r.name == ff.mismatch_logger.name
+ ]
+
+
+def _segment(key: str, rules: list[list[dict[str, Any]]], **members: Any):
+ return {
+ "key": key,
+ "name": key.title(),
+ "rules": [{"clauses": clauses} for clauses in rules],
+ **members,
+ }
+
+
+def _clause(attribute: str, op: str, *values: Any) -> dict[str, Any]:
+ # Without a context kind LaunchDarkly reads "/custom/role" as a literal name.
+ return {
+ "attribute": attribute,
+ "op": op,
+ "values": list(values),
+ "negate": False,
+ "contextKind": "user",
+ }
+
+
+def _in_segment(*keys: str) -> dict[str, Any]:
+ return _clause("segmentMatch", "segmentMatch", *keys)
+
+
+def _flag(
+ key: str,
+ variations: list[Any],
+ *,
+ on: bool = True,
+ off: int = 1,
+ targets: list[dict[str, Any]] | None = None,
+ rules: list[tuple[list[dict[str, Any]], int]] | None = None,
+ fallthrough: int = 1,
+ names: list[str] | None = None,
+) -> dict[str, Any]:
+ """A flag as LaunchDarkly's REST API exports it."""
+ return {
+ "key": key,
+ "name": key,
+ "kind": "boolean" if variations == [True, False] else "multivariate",
+ "variations": [
+ {"value": v, **({"name": names[i]} if names else {})}
+ for i, v in enumerate(variations)
+ ],
+ "environments": {
+ ENV: {
+ "on": on,
+ "offVariation": off,
+ "targets": targets or [],
+ "contextTargets": [],
+ "rules": [
+ {"clauses": clauses, "variation": variation}
+ for clauses, variation in rules or []
+ ],
+ "fallthrough": {"variation": fallthrough},
+ "prerequisites": [],
+ }
+ },
+ }
+
+
+USER_IDS = {"employee": str(uuid.uuid4()), "outsider": str(uuid.uuid4())}
+USER_IDS["targeted"] = TARGETED
+
+SEGMENTS = [
+ _segment(
+ "employee", [[_clause("email", "endsWith", "@agpt.co")]], included=[TARGETED]
+ ),
+ _segment("admin", [[_clause("/custom/role", "in", "admin")]]),
+]
+
+LIMITS = {"daily": 625000, "weekly": 3125000}
+MULTIPLIERS = {"MAX": 42.66, "PRO": 5}
+EXPORT = [
+ _flag("AutoMod", [True, False], on=False),
+ _flag("artifacts-page", [True, False], fallthrough=0),
+ _flag("copilot-voice-mode", [True, False], rules=[([_in_segment("employee")], 0)]),
+ _flag(
+ "chat-mode-option",
+ [True, False],
+ rules=[
+ ([_clause("email", "in", "[email protected]")], 0),
+ ([_in_segment("admin")], 0),
+ ],
+ ),
+ _flag(
+ "ai-agent-execution-summary",
+ [True, False],
+ targets=[{"values": [TARGETED], "variation": 0}],
+ rules=[([_clause("email_domain", "in", "agpt.co")], 0)],
+ ),
+ _flag("copilot-cost-limits", [LIMITS, {}], fallthrough=0),
+ _flag(
+ "copilot-tier-multipliers",
+ [MULTIPLIERS, {}],
+ rules=[([_in_segment("employee")], 1)],
+ fallthrough=0,
+ ),
+ _flag("copilot-model-routing", [{"fast": {"standard": "a/b"}}, {}], on=False),
+ _flag(
+ "copilot-tier-stripe-prices",
+ [{"PRO": "price_dev"}, {"PRO": "price_prod"}],
+ names=["dev", "prod"],
+ fallthrough=0,
+ ),
+ _flag("stripe-product-id-topup", ["", "prod_topup"], on=False),
+]
+ANONYMOUS_READS = [
+ "AutoMod",
+ "artifacts-page",
+ "copilot-cost-limits",
+ "copilot-tier-multipliers",
+ "copilot-model-routing",
+ "copilot-tier-stripe-prices",
+ "stripe-product-id-topup",
+]
+# What LaunchDarkly serves each reader, so a fixture that served everyone the
+# same could not pass for a port that ignores targeting.
+ANSWERS: dict[str, dict[str, Any]] = {
+ "AutoMod": {"*": False},
+ "artifacts-page": {"*": True},
+ "copilot-voice-mode": {"*": False, "employee": True, "targeted": True},
+ "chat-mode-option": {"*": False, "employee": True, "outsider": True},
+ "ai-agent-execution-summary": {"*": False, "employee": True, "targeted": True},
+ "copilot-cost-limits": {"*": LIMITS},
+ "copilot-tier-multipliers": {"*": MULTIPLIERS, "employee": {}, "targeted": {}},
+ "copilot-model-routing": {"*": {}},
+ "copilot-tier-stripe-prices": {"*": {"PRO": "price_dev"}},
+ "stripe-product-id-topup": {"*": "prod_topup"},
+}
+
+
+def _launchdarkly(tmp_path) -> LDClient:
+ """LaunchDarkly's own evaluator over the export, in its SDK data format."""
+ path = tmp_path / "launchdarkly.json"
+ path.write_text(
+ json.dumps(
+ {
+ "flags": {f["key"]: _sdk_flag(f) for f in EXPORT},
+ "segments": {s["key"]: _sdk_segment(s) for s in SEGMENTS},
+ }
+ )
+ )
+ client = LDClient(
+ LDConfig(
+ "sdk-test",
+ update_processor_class=Files.new_data_source(paths=[str(path)]),
+ send_events=False,
+ )
+ )
+ assert client.is_initialized()
+ return client
+
+
+def _sdk_flag(flag: dict[str, Any]) -> dict[str, Any]:
+ cfg = flag["environments"][ENV]
+ return {
+ **{k: cfg[k] for k in ("on", "offVariation", "targets", "contextTargets")},
+ "key": flag["key"],
+ "version": 1,
+ "salt": flag["key"],
+ "variations": [v["value"] for v in flag["variations"]],
+ "rules": [{"id": f"rule-{i}", **r} for i, r in enumerate(cfg["rules"])],
+ "fallthrough": cfg["fallthrough"],
+ "prerequisites": [],
+ }
+
+
+def _sdk_segment(segment: dict[str, Any]) -> dict[str, Any]:
+ return {
+ "key": segment["key"],
+ "version": 1,
+ "salt": segment["key"],
+ "included": segment.get("included", []),
+ "excluded": [],
+ "rules": [{"id": f"rule-{i}", **r} for i, r in enumerate(segment["rules"])],
+ }
+
+
+def _posthog() -> Posthog:
+ """PostHog's in-process evaluator over what the sync would write."""
+ cohorts = {s["key"]: map_segment(s, ENV) for s in SEGMENTS}
+ payloads = [c.payload for c in cohorts.values() if c.payload is not None]
+ ids = {p["name"]: i for i, p in enumerate(payloads, start=1)}
+ flags = []
+ for i, exported in enumerate(EXPORT, start=1):
+ mapped = map_flag(exported, ENV, cohorts)
+ assert mapped.payload is not None, mapped.decisions
+ filters = resolve_cohort_refs(mapped.payload["filters"], ids)
+ flags.append({**mapped.payload, "id": i, "filters": filters})
+
+ client = Posthog("phc-test", secret_key="phx-test", enable_local_evaluation=False)
+ client.feature_flags = flags
+ client.cohorts = {str(ids[p["name"]]): p["filters"]["properties"] for p in payloads}
+ client._get_flags_decision = _no_remote_flags
+ return client
+
+
+def _no_remote_flags(*args, **kwargs):
+ raise ConnectionError("only an in-process answer counts here")
+
+
+def _record_posthog_reads(monkeypatch) -> dict[str, tuple[Any, bool]]:
+ reads: dict[str, tuple[Any, bool]] = {}
+ evaluate = ph.evaluate_flag
+
+ async def recording(flag_key: str, *args: Any, **kwargs: Any):
+ reads[flag_key] = await evaluate(flag_key, *args, **kwargs)
+ return reads[flag_key]
+
+ monkeypatch.setattr(ph, "evaluate_flag", recording)
+ return reads
+
+
+class _AuthTable:
+ """The auth rows the backend builds both vendors' user context from."""
+
+ async def get_auth_user_flag_fields(self, user_id: str):
+ return {USER_IDS[who]: fields for who, fields in PEOPLE.items()}[user_id]
diff --git a/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsx b/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsx
index 2e0eb027c6..67599d5549 100644
--- a/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsx
+++ b/autogpt_platform/frontend/src/providers/posthog/__tests__/posthog-user-tracker.test.tsx
@@ -71,6 +71,7 @@ describe("PostHogUserTracker", () => {
render(<PostHogUserTracker />);
expect(posthog.setPersonPropertiesForFlags).toHaveBeenCalledWith({
+ email: "[email protected]",
email_domain: "example.com",
role: "authenticated",
created_at: "2026-05-08T12:00:00Z",
diff --git a/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.ts b/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.ts
index 927d4e7c23..898f360dc2 100644
--- a/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.ts
+++ b/autogpt_platform/frontend/src/services/feature-flags/__tests__/helpers.test.ts
@@ -103,8 +103,9 @@ describe("buildLDContext", () => {
});
describe("buildFlagPersonProperties", () => {
- it("carries the attributes the ported rules target on, never the raw email", () => {
+ it("carries the attributes the ported rules target on", () => {
expect(buildFlagPersonProperties(userFixture())).toEqual({
+ email: "[email protected]",
email_domain: "example.com",
role: "authenticated",
created_at: "2026-05-08T12:00:00Z",
@@ -116,6 +117,9 @@ describe("buildFlagPersonProperties", () => {
userFixture({ role: undefined, created_at: undefined }),
);
- expect(properties).toEqual({ email_domain: "example.com" });
+ expect(properties).toEqual({
+ email: "[email protected]",
+ email_domain: "example.com",
+ });
});
});
diff --git a/autogpt_platform/frontend/src/services/feature-flags/helpers.ts b/autogpt_platform/frontend/src/services/feature-flags/helpers.ts
index 15d4dea0dd..1febab8e25 100644
--- a/autogpt_platform/frontend/src/services/feature-flags/helpers.ts
+++ b/autogpt_platform/frontend/src/services/feature-flags/helpers.ts
@@ -32,11 +32,11 @@ export interface LDMultiContext {
export type LDContext = LDUserContext | LDMultiContext;
// The attributes the ported flag targeting rules match on, in the flat shape
-// PostHog person properties take. No raw email: no rule reads it, and
-// individual targets key on the distinct id.
+// PostHog person properties take; the synced employee cohort matches on email.
export function buildFlagPersonProperties(user: User): Record<string, string> {
return {
...(user.email && {
+ email: user.email,
email_domain: user.email.split("@").at(-1) ?? "",
}),
...(user.role && { role: user.role }),