Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(flags): reconstruct deleted flags in historical test evaluation (#124511)

ruby childs0384d0b67ab0
diff --git a/products/feature_flags/backend/api/feature_flag.py b/products/feature_flags/backend/api/feature_flag.py
index 807c80d6f125..faa5591f7dc0 100644
--- a/products/feature_flags/backend/api/feature_flag.py
+++ b/products/feature_flags/backend/api/feature_flag.py
@@ -323,6 +323,13 @@ EARLY_EXIT_FLAG = "feature-flag-early-exit"
ENFORCE_FEATURE_FLAG_WRITE_SCOPE_FLAG = "enforce-feature-flag-write-scope-cross-resource"
ENCRYPTED_VERSION_HISTORY_UNAVAILABLE = "Version history is not available for flags with encrypted payloads."
+ENCRYPTED_HISTORICAL_EVALUATION_UNAVAILABLE = (
+ "Historical evaluation is not available for flags with encrypted payloads."
+)
+TOMBSTONED_KEY_REUSED_UNAVAILABLE = (
+ "Historical evaluation is not available for this flag. Deleting it renamed its key, "
+ "and a newer flag now uses the original key, so the original bucketing can't be reproduced."
+)
# What a config version 2 write may change, in request-field names; everything else stays unsupported
# until the task that owns it lands. Disabling and soft-deleting are incident controls and need no admission.
@@ -5968,6 +5975,17 @@ class FeatureFlagViewSet(
evaluation_distinct_id = distinct_id if distinct_id and distinct_id in distinct_ids else sorted(distinct_ids)[0]
person_properties: dict[str, Any] = {}
+ # Mirror the versions endpoint: encrypted payloads are only served through
+ # /remote_config, which handles decryption. The reconstructed override would
+ # otherwise hand the flags service the stored ciphertext for a flag that the
+ # team's flag list deliberately excludes, and the evaluation response would
+ # echo it back as the payload.
+ if timestamp and feature_flag.has_encrypted_payloads:
+ return Response(
+ {"error": ENCRYPTED_HISTORICAL_EVALUATION_UNAVAILABLE},
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+
# Build person properties at timestamp if provided
if timestamp:
try:
@@ -6016,6 +6034,16 @@ class FeatureFlagViewSet(
timestamp=timestamp,
team_id=self.team_id,
)
+ # `has_encrypted_payloads` is mutable: a flag downgraded to plaintext keeps
+ # ciphertext in the activity log the reconstruction reads, so gate on the
+ # reconstructed version's own state too (same rule as the versions endpoint).
+ # Index rather than .get(): if a refactor ever drops the field from
+ # reconstruction, this gate must fail loud, not quietly open.
+ if reconstructed_flag_data["has_encrypted_payloads"]:
+ return Response(
+ {"error": ENCRYPTED_HISTORICAL_EVALUATION_UNAVAILABLE},
+ status=status.HTTP_400_BAD_REQUEST,
+ )
evaluation_filters = reconstructed_flag_data.get("filters") or {}
except VersionNotFound:
@@ -6042,6 +6070,24 @@ class FeatureFlagViewSet(
status=status.HTTP_500_INTERNAL_SERVER_ERROR,
)
+ # Bucketing hashes the flag key, and soft-deleting a flag linked to an
+ # experiment renames its key to "<key>:deleted:<id>". A historical
+ # evaluation must run under the original key, or users land in a
+ # different hash bucket than they did historically and get a wrong
+ # variant. If a live flag has reclaimed the original key, the flags
+ # service would match that flag by key and answer with its current
+ # state, so refuse instead.
+ evaluation_key = feature_flag.key
+ if timestamp and feature_flag.deleted:
+ original_key = feature_flag.key_without_tombstone()
+ if original_key != feature_flag.key:
+ if FeatureFlag.objects.filter(team_id=feature_flag.team_id, key=original_key).exists():
+ return Response(
+ {"error": TOMBSTONED_KEY_REUSED_UNAVAILABLE},
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+ evaluation_key = original_key
+
# Evaluate the flag
try:
# Get team API token for Rust service
@@ -6062,13 +6108,21 @@ class FeatureFlagViewSet(
flag_dict: dict[str, Any] = {
k: reconstructed_flag_data[k] for k in RUST_FLAG_FIELDS if k in reconstructed_flag_data
}
- flag_dict.update({"id": feature_flag.id, "key": feature_flag.key, "team_id": feature_flag.team_id})
+ flag_dict.update({"id": feature_flag.id, "key": evaluation_key, "team_id": feature_flag.team_id})
+
+ # Reconstruction succeeded, so find_version_at_timestamp proved the
+ # flag was not deleted at the timestamp. The reconstructed `deleted`
+ # can still be True: a config-v1 bulk delete sets deleted=True
+ # without a version bump and logs empty changes, so the walk-back
+ # never undoes it. The flags service skips deleted flags, so the
+ # override must ship deleted=False.
+ flag_dict["deleted"] = False
# Stringify datetime fields for JSON serialization.
if flag_dict.get("created_at") and hasattr(flag_dict["created_at"], "isoformat"):
flag_dict["created_at"] = flag_dict["created_at"].isoformat()
- override_definitions = {feature_flag.key: flag_dict}
+ override_definitions = {evaluation_key: flag_dict}
except Exception as e:
logger.exception("Failed to serialize flag for override")
capture_exception(e)
@@ -6091,7 +6145,7 @@ class FeatureFlagViewSet(
detailed_analysis=True,
person_properties=person_properties,
only_use_override_person_properties=timestamp is not None,
- flag_keys=[feature_flag.key],
+ flag_keys=[evaluation_key],
internal_request_token=internal_token,
override_flags_definitions=override_definitions,
# A pooled connection that the service closed fails once with a reset, so retry it.
@@ -6101,7 +6155,7 @@ class FeatureFlagViewSet(
# Extract the flag result from the Rust response
flags = rust_response.get("flags", {})
- flag_result = flags.get(feature_flag.key)
+ flag_result = flags.get(evaluation_key)
# Initialize defaults
condition_index = None
@@ -6126,6 +6180,26 @@ class FeatureFlagViewSet(
reason_description = reason_data.get("description") if reason_data else None
condition_index = reason_data.get("condition_index") if reason_data else None
payload = metadata.get("payload") if metadata else None
+ # A failed per-flag result also comes back without conditions.
+ # Split it by cause: a dependency the definition cannot resolve
+ # is deterministic, so a 400 tells the caller the request itself
+ # cannot succeed. Anything else (timeouts, connection
+ # exhaustion) is transient service trouble, so keep a logged
+ # 5xx that retries and alerting can see.
+ if timestamp and flag_result.get("failed"):
+ if reason.startswith(("dependency_not_found", "dependency_cycle")):
+ return Response(
+ {"error": f"Historical evaluation failed: {reason_description or reason}."},
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+ logger.error(
+ "Historical evaluation failed in the flags service",
+ extra={"flag_key": feature_flag.key, "reason": reason},
+ )
+ return Response(
+ {"error": f"Historical evaluation failed: {reason_description or reason}. Try again."},
+ status=status.HTTP_502_BAD_GATEWAY,
+ )
# Extract conditions from flag result (only valid path per Rust FlagDetails contract).
# When ``timestamp`` is set, missing conditions almost certainly means Rust
# rejected the internal token and silently fell back to current-state
diff --git a/products/feature_flags/backend/api/test/test_feature_flag.py b/products/feature_flags/backend/api/test/test_feature_flag.py
index f309488e8dd7..fd8369c7ab9b 100644
--- a/products/feature_flags/backend/api/test/test_feature_flag.py
+++ b/products/feature_flags/backend/api/test/test_feature_flag.py
@@ -14913,6 +14913,367 @@ class TestFeatureFlagTestEvaluation(APIBaseTest, ClickhouseTestMixin):
# by the (reconstructed) flag's conditions.
self.assertEqual(response.json()["person_properties"], {"email": "[email protected]"})
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_before_single_soft_delete_sends_undeleted_override(
+ self, mock_get_flags, _mock_build_props
+ ):
+ """A timestamp before a single soft delete must ship the reconstructed
+ definition with deleted=False in the override payload. The flags service
+ excludes deleted flags from the team's flag list, so this payload is the
+ only vehicle for the historical definition — a deleted=True (or dropped)
+ override surfaces as result=False, reason=flag_not_found in the UI
+ Testing tab and the MCP tool."""
+ filters = {"groups": [{"properties": [], "rollout_percentage": 100}]}
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="historical-deleted-flag",
+ filters=filters,
+ created_by=self.user,
+ version=1,
+ )
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ # Soft delete through the API so the real activity-logging path runs.
+ delete_response = self.client.patch(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",
+ {"deleted": True},
+ format="json",
+ )
+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())
+ flag.refresh_from_db()
+ self.assertTrue(flag.deleted)
+
+ mock_get_flags.return_value = {
+ "flags": {
+ "historical-deleted-flag": {
+ "enabled": True,
+ "variant": None,
+ "reason": {"code": "condition_match", "condition_index": 0},
+ "metadata": {},
+ "conditions": [],
+ }
+ }
+ }
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ sent = mock_get_flags.call_args.kwargs["override_flags_definitions"]["historical-deleted-flag"]
+ self.assertIs(sent["deleted"], False, f"override payload must undelete the flag, got {sent!r}")
+ self.assertEqual(sent["filters"], filters)
+ self.assertEqual(sent["team_id"], self.team.pk)
+ self.assertEqual(response.json()["result"], True)
+
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_before_bulk_delete_sends_undeleted_override(self, mock_get_flags, _mock_build_props):
+ """Bulk delete of a non-v2 flag writes deleted=True via .update() without a
+ version bump and logs activity="deleted" with empty changes. Reconstruction
+ starts from the current (deleted) row and the walk-back never undoes the
+ deletion, so without a correction the override payload ships deleted=True
+ and the flags service skips the flag. The endpoint must force
+ deleted=False: a successful reconstruction already proves the flag existed
+ and was not deleted at the timestamp."""
+ filters = {"groups": [{"properties": [], "rollout_percentage": 100}]}
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="bulk-deleted-flag",
+ filters=filters,
+ created_by=self.user,
+ version=1,
+ )
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ delete_response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/bulk_delete/",
+ {"ids": [flag.id]},
+ format="json",
+ )
+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())
+ flag.refresh_from_db()
+ self.assertTrue(flag.deleted)
+
+ mock_get_flags.return_value = {
+ "flags": {
+ "bulk-deleted-flag": {
+ "enabled": True,
+ "variant": None,
+ "reason": {"code": "condition_match", "condition_index": 0},
+ "metadata": {},
+ "conditions": [],
+ }
+ }
+ }
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ sent = mock_get_flags.call_args.kwargs["override_flags_definitions"]["bulk-deleted-flag"]
+ self.assertIs(sent["deleted"], False, f"override payload must undelete the flag, got {sent!r}")
+ self.assertEqual(sent["filters"], filters)
+ self.assertEqual(response.json()["result"], True)
+
+ # A timestamp after the deletion must still refuse: with deleted=False
+ # forced onto the override, this 400 is the only thing stopping a
+ # deleted flag from evaluating as live.
+ after_response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": now().isoformat()},
+ format="json",
+ )
+ self.assertEqual(after_response.status_code, status.HTTP_400_BAD_REQUEST, after_response.json())
+ self.assertIn("did not exist", after_response.json()["error"])
+ self.assertEqual(mock_get_flags.call_count, 1)
+
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_with_timestamp_rejects_encrypted_payload_flags(self, mock_get_flags, mock_build_props):
+ """The reconstructed override would carry the stored ciphertext for a flag the
+ team's flag list deliberately excludes, so historical evaluation must refuse
+ encrypted-payload flags the same way the versions endpoint does. The first
+ gate fires before any ClickHouse or flags-service work."""
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="encrypted-flag",
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 100}],
+ "payloads": {"true": "ciphertext"},
+ },
+ has_encrypted_payloads=True,
+ created_by=self.user,
+ version=1,
+ )
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)
+ self.assertIn("encrypted payloads", response.json()["error"])
+ mock_build_props.assert_not_called()
+ mock_get_flags.assert_not_called()
+
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_rejects_flag_encrypted_at_timestamp(self, mock_get_flags, _mock_build_props):
+ """The second encrypted gate: a flag that is plaintext now but had
+ encrypted payloads at the timestamp must still be refused, or the
+ reconstruction would ship the activity log's stored ciphertext to the
+ flags service and echo it back as the payload."""
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ created_by=self.user,
+ key="downgraded-flag",
+ filters={
+ "groups": [{"properties": [], "rollout_percentage": 100}],
+ "payloads": {"true": "ciphertext"},
+ },
+ is_remote_configuration=True,
+ has_encrypted_payloads=True,
+ version=1,
+ )
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ # Downgrade through the API so the change is activity-logged.
+ downgrade = self.client.patch(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",
+ {
+ "has_encrypted_payloads": False,
+ "is_remote_configuration": False,
+ "filters": {"groups": [{"properties": [], "rollout_percentage": 100}], "payloads": {}},
+ },
+ format="json",
+ )
+ self.assertEqual(downgrade.status_code, status.HTTP_200_OK, downgrade.json())
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())
+ self.assertIn("encrypted payloads", response.json()["error"])
+ mock_get_flags.assert_not_called()
+
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_deleted_experiment_flag_uses_original_key(self, mock_get_flags, _mock_build_props):
+ """Deleting a flag linked to an experiment renames its key to
+ "<key>:deleted:<id>". Bucketing hashes the key, so the override must
+ carry the original key or historical evaluation reports the wrong
+ bucket (and on a multivariate split, the wrong variant)."""
+ filters = {"groups": [{"properties": [], "rollout_percentage": 50}]}
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="exp-flag",
+ filters=filters,
+ created_by=self.user,
+ version=1,
+ )
+ create_experiment(team_id=self.team.id, feature_flag_id=flag.id, created_by_id=self.user.id, deleted=True)
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ delete_response = self.client.patch(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",
+ {"deleted": True},
+ format="json",
+ )
+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())
+ flag.refresh_from_db()
+ self.assertEqual(flag.key, f"exp-flag:deleted:{flag.id}")
+
+ mock_get_flags.return_value = {
+ "flags": {
+ "exp-flag": {
+ "enabled": True,
+ "variant": None,
+ "reason": {"code": "condition_match", "condition_index": 0},
+ "metadata": {},
+ "conditions": [],
+ }
+ }
+ }
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())
+ overrides = mock_get_flags.call_args.kwargs["override_flags_definitions"]
+ self.assertEqual(list(overrides.keys()), ["exp-flag"])
+ self.assertEqual(overrides["exp-flag"]["key"], "exp-flag")
+ self.assertEqual(mock_get_flags.call_args.kwargs["flag_keys"], ["exp-flag"])
+ self.assertEqual(response.json()["result"], True)
+
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_tombstoned_key_reclaimed_by_live_flag_refuses(self, mock_get_flags, _mock_build_props):
+ """When a live flag has reclaimed a deleted flag's original key, the
+ flags service would match the live flag by key and answer with its
+ current state, which would read as a historical result. Refuse instead."""
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="reclaimed-flag",
+ filters={"groups": [{"properties": [], "rollout_percentage": 50}]},
+ created_by=self.user,
+ version=1,
+ )
+ create_experiment(team_id=self.team.id, feature_flag_id=flag.id, created_by_id=self.user.id, deleted=True)
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ delete_response = self.client.patch(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",
+ {"deleted": True},
+ format="json",
+ )
+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())
+ FeatureFlag.objects.create(team=self.team, key="reclaimed-flag", created_by=self.user, version=1)
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())
+ self.assertIn("original key", response.json()["error"])
+ mock_get_flags.assert_not_called()
+
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_failed_flag_result_returns_evaluation_error(self, mock_get_flags, _mock_build_props):
+ """A failed per-flag result (e.g. a cohort the service could not
+ resolve) must surface as an evaluation error, not as the 502 that
+ blames INTERNAL_REQUEST_TOKEN configuration."""
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="failing-flag",
+ filters={"groups": [{"properties": [], "rollout_percentage": 100}]},
+ created_by=self.user,
+ version=1,
+ )
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ mock_get_flags.return_value = {
+ "flags": {
+ "failing-flag": {
+ "enabled": False,
+ "failed": True,
+ "variant": None,
+ "reason": {"code": "dependency_not_found_cohort", "description": "Cohort dependency not found"},
+ "metadata": {},
+ }
+ }
+ }
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())
+ self.assertIn("Cohort dependency not found", response.json()["error"])
+
+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})
+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")
+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")
+ def test_test_evaluation_transient_failure_returns_502(self, mock_get_flags, _mock_build_props):
+ """A transient service failure (timeout, connection exhaustion) is not a
+ property of the request, so it keeps a 5xx that retries and alerting
+ can see, unlike the deterministic dependency 400 above."""
+ flag = FeatureFlag.objects.create(
+ team=self.team,
+ key="transient-failing-flag",
+ filters={"groups": [{"properties": [], "rollout_percentage": 100}]},
+ created_by=self.user,
+ version=1,
+ )
+ create_person(team=self.team, distinct_ids=["test-user"])
+
+ mock_get_flags.return_value = {
+ "flags": {
+ "transient-failing-flag": {
+ "enabled": False,
+ "failed": True,
+ "variant": None,
+ "reason": {"code": "group_mapping_retry", "description": "Failed to fetch group mapping"},
+ "metadata": {},
+ }
+ }
+ }
+
+ response = self.client.post(
+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",
+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},
+ format="json",
+ )
+
+ self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY, response.json())
+ self.assertIn("Failed to fetch group mapping", response.json()["error"])
+
def test_test_evaluation_distinct_id_person_id_conflict(self):
"""Test validation error when both distinct_id and person_id are provided."""
flag = FeatureFlag.objects.create(team=self.team, key="test-flag")
diff --git a/products/feature_flags/backend/test/test_version_history.py b/products/feature_flags/backend/test/test_version_history.py
index b56cf489faa2..5ccb88e9709a 100644
--- a/products/feature_flags/backend/test/test_version_history.py
+++ b/products/feature_flags/backend/test/test_version_history.py
@@ -467,6 +467,111 @@ class TestTimestampBasedReconstruction(TestReconstructFlagAtVersion):
version = find_version_at_timestamp(flag, current_timestamp, self.team.id)
assert version is None
+ def test_find_version_at_timestamp_deleted_flag_with_later_versioned_edit(self):
+ """A versioned edit logged after the deletion must not mask it: the flag
+ was still deleted at that time, so the version lookup returns None."""
+ flag = self._create_flag(name="V1", version=1)
+
+ log_activity(
+ organization_id=self.organization.id,
+ team_id=self.team.id,
+ user=self.user,
+ was_impersonated=False,
+ item_id=flag.id,
+ scope="FeatureFlag",
+ activity="deleted",
+ detail=Detail(
+ changes=[Change(type="FeatureFlag", action="changed", field="version", before=1, after=2)],
+ name=flag.key,
+ ),
+ )
+ # A write to the deleted flag's row that logs a versioned "updated" entry.
+ log_activity(
+ organization_id=self.organization.id,
+ team_id=self.team.id,
+ user=self.user,
+ was_impersonated=False,
+ item_id=flag.id,
+ scope="FeatureFlag",
+ activity="updated",
+ detail=Detail(
+ changes=[Change(type="FeatureFlag", action="changed", field="version", before=2, after=3)],
+ name=flag.key,
+ ),
+ )
+
+ version = find_version_at_timestamp(flag, flag.created_at + timedelta(hours=1), self.team.id)
+ assert version is None
+
+ def test_find_version_at_timestamp_legacy_deletion_logged_as_updated(self):
+ """Before #41043 (2025-11-19), a soft delete through PATCH was logged
+ as activity="updated" with a `deleted` False-to-True change. Those
+ entries still decide existence: a timestamp after one must return None,
+ or the endpoint's forced deleted=False revives the flag."""
+ flag = self._create_flag(name="V1", version=1)
+
+ log_activity(
+ organization_id=self.organization.id,
+ team_id=self.team.id,
+ user=self.user,
+ was_impersonated=False,
+ item_id=flag.id,
+ scope="FeatureFlag",
+ activity="updated",
+ detail=Detail(
+ changes=[
+ Change(type="FeatureFlag", action="changed", field="deleted", before=False, after=True),
+ Change(type="FeatureFlag", action="changed", field="version", before=1, after=2),
+ ],
+ name=flag.key,
+ ),
+ )
+
+ version = find_version_at_timestamp(flag, flag.created_at + timedelta(hours=1), self.team.id)
+ assert version is None
+
+ def test_find_version_at_timestamp_legacy_restore_logged_as_updated(self):
+ """The legacy counterpart of a restore: an activity="updated" entry
+ with a `deleted` True-to-False change confirms the flag was alive, so
+ an older legacy deletion must not hide the version."""
+ flag = self._create_flag(name="V1", version=1)
+
+ log_activity(
+ organization_id=self.organization.id,
+ team_id=self.team.id,
+ user=self.user,
+ was_impersonated=False,
+ item_id=flag.id,
+ scope="FeatureFlag",
+ activity="updated",
+ detail=Detail(
+ changes=[
+ Change(type="FeatureFlag", action="changed", field="deleted", before=False, after=True),
+ Change(type="FeatureFlag", action="changed", field="version", before=1, after=2),
+ ],
+ name=flag.key,
+ ),
+ )
+ log_activity(
+ organization_id=self.organization.id,
+ team_id=self.team.id,
+ user=self.user,
+ was_impersonated=False,
+ item_id=flag.id,
+ scope="FeatureFlag",
+ activity="updated",
+ detail=Detail(
+ changes=[
+ Change(type="FeatureFlag", action="changed", field="deleted", before=True, after=False),
+ Change(type="FeatureFlag", action="changed", field="version", before=2, after=3),
+ ],
+ name=flag.key,
+ ),
+ )
+
+ version = find_version_at_timestamp(flag, flag.created_at + timedelta(hours=1), self.team.id)
+ assert version == 3
+
def test_find_version_at_timestamp_soft_deleted_then_restored(self):
"""Test that restored flags return proper version information."""
flag = self._create_flag(name="V1", version=1)
diff --git a/products/feature_flags/backend/version_history.py b/products/feature_flags/backend/version_history.py
index 4adbe1d616fa..6fe4709af8c9 100644
--- a/products/feature_flags/backend/version_history.py
+++ b/products/feature_flags/backend/version_history.py
@@ -1,6 +1,7 @@
from datetime import datetime
from typing import Any
+from django.db.models import Q
from django.utils import timezone
from posthog.models.activity_logging.activity_log import ActivityLog, common_field_exclusions, field_exclusions
@@ -209,35 +210,51 @@ def find_version_at_timestamp(flag: FeatureFlag, timestamp: datetime, team_id: i
if timestamp < flag.created_at:
return None
- # Get activity log entries at or before the timestamp, newest first
- entries = (
- ActivityLog.objects.filter(
- team_id=team_id,
- scope="FeatureFlag",
- item_id=str(flag.id),
- activity__in=["updated", "deleted", "restored"],
- created_at__lte=timestamp,
- )
- .order_by("-created_at", "-id")
- .values_list("activity", "detail")[:MAX_HISTORY_ENTRIES]
- .iterator()
+ history = ActivityLog.objects.filter(
+ team_id=team_id,
+ scope="FeatureFlag",
+ item_id=str(flag.id),
+ created_at__lte=timestamp,
+ ).order_by("-created_at", "-id")
+
+ # Deletion state and version are resolved independently. The most recent
+ # entry that changed deletion state at or before the timestamp decides
+ # existence, while a versioned "updated" entry does not, since a deleted
+ # flag's row can still receive versioned writes (and bulk deletes log
+ # "deleted" with empty changes). Deletion state comes from the activity
+ # name AND from a `deleted` field change: before #41043 (2025-11-19), soft
+ # deletes and restores through PATCH were logged as "updated" with a
+ # `deleted` change, and those legacy entries still decide existence.
+ lifecycle_entry = (
+ history.filter(Q(activity__in=["deleted", "restored"]) | Q(detail__changes__contains=[{"field": "deleted"}]))
+ .values_list("activity", "detail")
+ .first()
)
-
- for activity, detail in entries:
+ if lifecycle_entry is not None:
+ activity, detail = lifecycle_entry
changes = (detail or {}).get("changes") or []
- version_after = _get_version_after(changes)
-
- # Track deletion/restoration regardless of whether this entry carried a
- # version change. Bulk deletes are logged with empty changes.
- if activity == "deleted":
+ deleted_after = next(
+ (change.get("after") for change in changes if change.get("field") == "deleted"),
+ None,
+ )
+ marks_deleted = activity == "deleted" or bool(deleted_after)
+ if marks_deleted:
return None
+ # The most recent entry with a version change decides the version.
+ details = (
+ history.filter(activity__in=["updated", "deleted", "restored"])
+ .values_list("detail", flat=True)[:MAX_HISTORY_ENTRIES]
+ .iterator()
+ )
+ for detail in details:
+ version_after = _get_version_after((detail or {}).get("changes") or [])
if version_after is not None:
return version_after
- # No activity log entries found. If timestamp is after the flag's last update,
- # return the current version. The scan has authoritative deletion state,
- # or we wouldn't reach here.
+ # No versioned entries found. If timestamp is after the flag's last update,
+ # return the current version. The lifecycle lookup above has authoritative
+ # deletion state, or we wouldn't reach here.
if timestamp >= (flag.updated_at or flag.created_at):
return flag.version
diff --git a/rust/feature-flags/src/handler/flags.rs b/rust/feature-flags/src/handler/flags.rs
index 6900fe6268fe..bcd9b5850814 100644
--- a/rust/feature-flags/src/handler/flags.rs
+++ b/rust/feature-flags/src/handler/flags.rs
@@ -4,9 +4,10 @@ use crate::{
types::{FlagsQueryParams, FlagsResponse},
},
flags::{
+ cache_builder::compute_flag_dependencies_or_single_stage,
feature_flag_list::PreparedFlags,
flag_analytics::SURVEY_TARGETING_FLAG_PREFIX,
- flag_models::{FeatureFlag, FeatureFlagList},
+ flag_models::{EvaluationMetadata, FeatureFlag, FeatureFlagList},
flag_service::FlagService,
},
utils::user_agent::{RuntimeType, UserAgentInfo},
@@ -141,23 +142,25 @@ fn collect_excluded_by_runtime(
/// are dropped with a warning — a caller must not be able to swap one flag's
/// identity for another (potentially cross-team) by submitting a forged
/// override payload.
+///
+/// An override whose key is absent from ``flags`` is ADDED rather than
+/// dropped: historical test evaluation reconstructs soft-deleted flags, and
+/// every source of the team's flag list (hypercache and PG) excludes deleted
+/// flags, so the reconstructed definition can only arrive this way. (The list
+/// also omits disabled flags that no other flag references, so those take the
+/// add path too when evaluated historically.) The added definition must carry
+/// the request's own ``team_id``: overrides only reach this path through
+/// internal-token requests, but the guard keeps a forged payload from
+/// evaluating another team's flag definition.
fn apply_flag_overrides(
- flags: &mut [FeatureFlag],
+ flags: &mut Vec<FeatureFlag>,
override_defs: &HashMap<String, Value>,
+ request_team_id: TeamId,
) -> Vec<String> {
let mut overridden_keys = Vec::new();
tracing::debug!("Processing {} override definitions", override_defs.len());
for (flag_key, override_def) in override_defs {
tracing::debug!("Processing override for flag: {}", flag_key);
- let Some(flag) = flags.iter_mut().find(|f| &f.key == flag_key) else {
- tracing::warn!("Flag not found for override: {}", flag_key);
- continue;
- };
- tracing::trace!(
- "Found flag to override: {}, current filters: {:?}",
- flag_key,
- flag.filters
- );
let override_flag = match serde_json::from_value::<FeatureFlag>(override_def.clone()) {
Ok(parsed) => parsed,
Err(e) => {
@@ -170,32 +173,49 @@ fn apply_flag_overrides(
continue;
}
};
- // Identity guard: a caller must not be able to use the override to
- // swap in another flag's identity. The override is keyed by flag_key
- // alone, so we reject payloads whose embedded key/id/team_id disagree
- // with the flag we resolved from the team's flag list.
- if override_flag.key != *flag_key
- || override_flag.id != flag.id
- || override_flag.team_id != flag.team_id
- {
+ // Identity guards shared by both paths: the override map is keyed by
+ // flag_key alone, so the payload's embedded key must match it, and the
+ // payload must belong to the requesting team (every flag in `flags`
+ // does, since both list sources filter by team).
+ if override_flag.key != *flag_key || override_flag.team_id != request_team_id {
tracing::warn!(
- "Override identity mismatch for flag {}: skipping (override key={:?}, id={}, team_id={}; expected key={:?}, id={}, team_id={})",
+ "Override identity mismatch for flag {}: skipping (override key={:?}, team_id={}; request team_id={})",
flag_key,
override_flag.key,
- override_flag.id,
override_flag.team_id,
- flag.key,
- flag.id,
- flag.team_id
+ request_team_id
);
continue;
}
- tracing::trace!(
- "Successfully parsed override flag: {}, new filters: {:?}",
- flag_key,
- override_flag.filters
- );
- *flag = override_flag;
+ match flags.iter().position(|f| &f.key == flag_key) {
+ Some(index) => {
+ // A payload must not swap in another flag's identity.
+ if override_flag.id != flags[index].id {
+ tracing::warn!(
+ "Override for flag {} has id {}, expected {}: skipping",
+ flag_key,
+ override_flag.id,
+ flags[index].id
+ );
+ continue;
+ }
+ flags[index] = override_flag;
+ }
+ None => {
+ // The key is not in the team's list, so this is a reconstructed
+ // deleted flag: add it. Evaluation stages are keyed by id, so
+ // the added flag must not reuse an existing flag's id.
+ if flags.iter().any(|f| f.id == override_flag.id) {
+ tracing::warn!(
+ "Override for flag {} reuses id {} of an existing flag: skipping",
+ flag_key,
+ override_flag.id
+ );
+ continue;
+ }
+ flags.push(override_flag);
+ }
+ }
overridden_keys.push(flag_key.clone());
}
overridden_keys
@@ -219,24 +239,54 @@ pub async fn fetch_and_filter(
// Apply override flag definitions if provided. Overrides require a clone
// because `prepared.flags` is Arc-backed and shared across requests.
- let overridden_flags: Option<PreparedFlags> = match override_flags_definitions {
- Some(override_defs) => {
- let mut flags_vec: Vec<FeatureFlag> = prepared.flags.iter().cloned().collect();
- let overridden_keys = apply_flag_overrides(&mut flags_vec, override_defs);
- if !overridden_keys.is_empty() {
- with_canonical_log(|log| {
- log.flags_overridden = Some(overridden_keys);
- });
+ // `overridden` is Some only when an override actually applied; a request
+ // whose overrides were all rejected falls back to the cached list. The
+ // cached evaluation metadata was computed from the live definitions: an
+ // ADDED flag (a reconstructed soft-deleted flag) sits in no dependency
+ // stage, so it would never evaluate and the staging math would miscount
+ // it as a cycle participant, while a REPLACED flag's historical version
+ // can depend on flags its live version does not, so the cached
+ // `transitive_deps` would leave those dependencies unevaluated.
+ // Recomputing from the merged list keeps metadata and flag list
+ // consistent. A dependency of an added flag that is itself still deleted
+ // stays absent from the list, so the recompute marks the added flag as
+ // missing a dependency and it fails closed, same as any unresolvable
+ // dependency.
+ let overridden: Option<(PreparedFlags, Arc<EvaluationMetadata>)> =
+ match override_flags_definitions {
+ Some(override_defs) => {
+ let mut flags_vec: Vec<FeatureFlag> = prepared.flags.iter().cloned().collect();
+ let overridden_keys = apply_flag_overrides(&mut flags_vec, override_defs, team_id);
+ if overridden_keys.is_empty() {
+ None
+ } else {
+ with_canonical_log(|log| {
+ log.flags_overridden = Some(overridden_keys);
+ });
+ // `has_unparseable_flags` is read-time state (set when the
+ // cache read skipped or omitted a flag), not derivable from
+ // the flag list, so the recompute must carry it over or a
+ // partial evaluation would stop reporting
+ // errorsWhileComputingFlags.
+ let metadata = Arc::new(EvaluationMetadata {
+ has_unparseable_flags: prepared.evaluation_metadata.has_unparseable_flags,
+ ..compute_flag_dependencies_or_single_stage(team_id, &flags_vec)
+ });
+ Some((PreparedFlags::seal(flags_vec), metadata))
+ }
}
- Some(PreparedFlags::seal(flags_vec))
- }
- None => None,
+ None => None,
+ };
+ let overrides_applied = overridden.is_some();
+ let (prepared_flags, evaluation_metadata) = match overridden {
+ Some((flags, metadata)) => (flags, metadata),
+ None => (
+ PreparedFlags::from_arc(Arc::clone(prepared.flags.as_arc())),
+ Arc::clone(&prepared.evaluation_metadata),
+ ),
};
- let flags: &[FeatureFlag] = match &overridden_flags {
- Some(p) => p,
- None => &prepared.flags,
- };
+ let flags: &[FeatureFlag] = &prepared_flags;
// Build the filtered-out set: user-disabled, deleted, survey filter, runtime/tag mismatches.
// This is the single source of truth for "should this flag be skipped during evaluation."
@@ -274,13 +324,20 @@ pub async fn fetch_and_filter(
// of refcount bumps rather than a deep copy of the flag slice, the
// `EvaluationMetadata` map, or the cohort vec.
let flag_list = FeatureFlagList {
- flags: match overridden_flags {
- Some(p) => p,
- None => PreparedFlags::from_arc(Arc::clone(prepared.flags.as_arc())),
- },
+ flags: prepared_flags,
filtered_out_flag_ids,
- evaluation_metadata: Arc::clone(&prepared.evaluation_metadata),
- cohorts: prepared.cohorts.as_ref().map(Arc::clone),
+ evaluation_metadata,
+ // The cached cohort list only holds cohorts the team's live flags
+ // reference. An override can name others (a reconstructed deleted
+ // flag's cohort, or one an older version of a live flag targeted), and
+ // the matcher treats a present list as complete, so a missing cohort
+ // fails evaluation with DependencyNotFound. Dropping the list makes
+ // the matcher load the team's full cohort set instead.
+ cohorts: if overrides_applied {
+ None
+ } else {
+ prepared.cohorts.as_ref().map(Arc::clone)
+ },
};
Ok(flag_list)
}
@@ -979,7 +1036,7 @@ mod tests {
let mut overrides = HashMap::new();
overrides.insert("my-flag".to_string(), override_payload(1, 100, "my-flag"));
- let applied = apply_flag_overrides(&mut flags, &overrides);
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
assert_eq!(applied, vec!["my-flag".to_string()]);
assert_eq!(flags[0].filters.groups[0].rollout_percentage, Some(50.0));
@@ -993,7 +1050,7 @@ mod tests {
// caller from swapping in another team's identity.
overrides.insert("my-flag".to_string(), override_payload(1, 999, "my-flag"));
- let applied = apply_flag_overrides(&mut flags, &overrides);
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
assert!(applied.is_empty(), "mismatched team_id must not be applied");
assert_eq!(flags[0].team_id, 100, "original team_id must be preserved");
@@ -1005,7 +1062,7 @@ mod tests {
let mut overrides = HashMap::new();
overrides.insert("my-flag".to_string(), override_payload(999, 100, "my-flag"));
- let applied = apply_flag_overrides(&mut flags, &overrides);
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
assert!(applied.is_empty(), "mismatched id must not be applied");
assert_eq!(flags[0].id, 1, "original id must be preserved");
@@ -1022,22 +1079,107 @@ mod tests {
override_payload(1, 100, "other-flag"),
);
- let applied = apply_flag_overrides(&mut flags, &overrides);
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
assert!(applied.is_empty(), "mismatched key must not be applied");
}
#[test]
- fn test_apply_flag_overrides_skips_unknown_keys() {
- let mut flags = vec![flag_with_team(1, 100, "my-flag")];
+ fn test_apply_flag_overrides_adds_flag_missing_from_team_list() {
+ // Historical test evaluation reconstructs a soft-deleted flag and ships it
+ // as an override. Every source of the team's flag list (cache and PG) excludes
+ // deleted flags, so the key is absent from `flags` and the override must be
+ // ADDED, not dropped, or /test_evaluation reports flag_not_found for
+ // timestamps at which the flag existed.
+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];
+ let mut overrides = HashMap::new();
+ overrides.insert(
+ "deleted-flag".to_string(),
+ override_payload(2, 100, "deleted-flag"),
+ );
+
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
+
+ assert_eq!(
+ applied,
+ vec!["deleted-flag".to_string()],
+ "override for a key absent from the team's flag list must be applied"
+ );
+ assert_eq!(
+ flags.len(),
+ 2,
+ "the override definition must be added to the flag list"
+ );
+ let added = flags
+ .iter()
+ .find(|f| f.key == "deleted-flag")
+ .expect("added flag must be present");
+ assert_eq!(added.id, 2);
+ assert_eq!(
+ added.team_id, 100,
+ "an added override must belong to the same team as the request"
+ );
+ assert_eq!(added.filters.groups[0].rollout_percentage, Some(50.0));
+ }
+
+ #[test]
+ fn test_apply_flag_overrides_rejects_added_flag_for_other_team() {
+ // An override for a key absent from the list may only be added when its
+ // team_id matches the requesting team, otherwise a forged payload could
+ // evaluate another team's flag definition.
+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];
let mut overrides = HashMap::new();
overrides.insert(
- "not-in-list".to_string(),
- override_payload(1, 100, "not-in-list"),
+ "deleted-flag".to_string(),
+ override_payload(2, 999, "deleted-flag"),
);
- let applied = apply_flag_overrides(&mut flags, &overrides);
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
- assert!(applied.is_empty());
+ assert!(applied.is_empty(), "cross-team override must not be added");
+ assert_eq!(flags.len(), 1, "flag list must be unchanged");
+ }
+
+ #[test]
+ fn test_apply_flag_overrides_rejects_added_flag_with_duplicate_id() {
+ // An added override must not reuse an existing flag's id: the
+ // evaluation stages are keyed by id, so a duplicate would corrupt
+ // staging and filtering.
+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];
+ let mut overrides = HashMap::new();
+ overrides.insert(
+ "deleted-flag".to_string(),
+ override_payload(1, 100, "deleted-flag"),
+ );
+
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
+
+ assert!(
+ applied.is_empty(),
+ "duplicate-id override must not be added"
+ );
+ assert_eq!(flags.len(), 1, "flag list must be unchanged");
+ }
+
+ #[test]
+ fn test_apply_flag_overrides_rejects_added_flag_with_mismatched_payload_key() {
+ // An added override's payload key must equal the map key it arrived
+ // under. Without this guard, a payload claiming an existing flag's key
+ // would be pushed next to the real flag, and the key-indexed response
+ // would silently shadow one of them.
+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];
+ let mut overrides = HashMap::new();
+ overrides.insert(
+ "deleted-flag".to_string(),
+ override_payload(2, 100, "live-flag"),
+ );
+
+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);
+
+ assert!(
+ applied.is_empty(),
+ "override whose payload key differs from its map key must not be added"
+ );
+ assert_eq!(flags.len(), 1, "flag list must be unchanged");
}
}
diff --git a/rust/feature-flags/src/handler/tests.rs b/rust/feature-flags/src/handler/tests.rs
index 379c874ee568..0d4c67d23a0f 100644
--- a/rust/feature-flags/src/handler/tests.rs
+++ b/rust/feature-flags/src/handler/tests.rs
@@ -39,7 +39,8 @@ use crate::{
test_utils::{
flag_list_with_metadata_and_filter, insert_flags_for_team_in_redis,
mock_group_type_cache, setup_hypercache_reader, setup_pg_reader_client,
- setup_pg_writer_client, setup_redis_client, setup_team_hypercache_reader, TestContext,
+ setup_pg_writer_client, setup_redis_client, setup_team_hypercache_reader,
+ write_flags_wire_json_to_redis, TestContext,
},
},
};
@@ -1499,6 +1500,477 @@ async fn test_fetch_and_filter_preserves_evaluation_metadata() {
assert!(ctx.transitive_deps.contains_key(&2));
}
+/// Historical test evaluation ships a reconstructed soft-deleted flag as an
+/// override. The team's cached flag list excludes deleted flags, so the
+/// override's key is absent from it: `fetch_and_filter` must add the override
+/// to the list, leave it un-filtered, and stage it in the evaluation metadata
+/// (a flag absent from `dependency_stages` is never evaluated and would be
+/// miscounted as a cycle participant).
+#[tokio::test]
+async fn test_fetch_and_filter_override_adds_flag_missing_from_team_list() {
+ let redis_client = setup_redis_client(None).await;
+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);
+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
+ let flag_service = FlagService::new(
+ redis_client.clone(),
+ reader.clone(),
+ team_hypercache_reader,
+ hypercache_reader,
+ Arc::new(FlagDefinitionsCache::disabled()),
+ NegativeCache::new(100, 300),
+ false,
+ );
+ let context = TestContext::new(None).await;
+ let team = context.insert_new_team(None).await.unwrap();
+
+ // The team's list carries only the live flag; the soft-deleted flag (id 2)
+ // is excluded at cache-write time, exactly like production.
+ let flags = vec![mock!(FeatureFlag,
+ name: "Live flag".mock_into(),
+ id: 1,
+ key: "live-flag".mock_into(),
+ team_id: team.id,
+ filters: FlagFilters::default()
+ )];
+ let eval_metadata = EvaluationMetadata {
+ dependency_stages: vec![vec![1]],
+ flags_with_missing_deps: vec![],
+ transitive_deps: HashMap::from([(1, std::collections::HashSet::new())]),
+ ..Default::default()
+ };
+ let wrapper = HypercacheFlagsWrapper {
+ flags: flags.clone(),
+ evaluation_metadata: eval_metadata,
+ cohorts: None,
+ };
+ let json_string = serde_json::to_string(&wrapper).unwrap();
+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)
+ .await
+ .unwrap();
+
+ let override_def = json!({
+ "id": 2,
+ "team_id": team.id,
+ "name": "Reconstructed deleted flag",
+ "key": "deleted-flag",
+ "filters": { "groups": [{"properties": [], "rollout_percentage": 100}] },
+ "deleted": false,
+ "active": true,
+ });
+ let overrides: HashMap<String, Value> =
+ HashMap::from([("deleted-flag".to_string(), override_def)]);
+
+ let query_params = FlagsQueryParams::default();
+ let result = fetch_and_filter(
+ &flag_service,
+ team.id,
+ &query_params,
+ &HeaderMap::new(),
+ None,
+ None,
+ Some(&overrides),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(
+ result.flags.len(),
+ 2,
+ "override for a key absent from the team's list must be added, not dropped"
+ );
+ let added = result
+ .flags
+ .iter()
+ .find(|f| f.key == "deleted-flag")
+ .expect("added flag must be in the list");
+ assert!(
+ !result.filtered_out_flag_ids.contains(&added.id),
+ "added flag must not be filtered out"
+ );
+ let staged: Vec<i32> = result
+ .evaluation_metadata
+ .dependency_stages
+ .iter()
+ .flatten()
+ .copied()
+ .collect();
+ assert!(
+ staged.contains(&added.id),
+ "added flag must appear in a dependency stage or it is never evaluated (stages: {staged:?})"
+ );
+}
+
+/// When an override adds a flag, `fetch_and_filter` recomputes the evaluation
+/// metadata. The recompute must keep the read-time `has_unparseable_flags`
+/// marker, or a partial evaluation would stop reporting
+/// `errorsWhileComputingFlags`.
+#[tokio::test]
+async fn test_fetch_and_filter_override_add_preserves_unparseable_marker() {
+ use crate::flags::test_helpers::payload_with_unparseable_flag;
+
+ let redis_client = setup_redis_client(None).await;
+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);
+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
+ let flag_service = FlagService::new(
+ redis_client.clone(),
+ reader.clone(),
+ team_hypercache_reader,
+ hypercache_reader,
+ Arc::new(FlagDefinitionsCache::disabled()),
+ NegativeCache::new(100, 300),
+ false,
+ );
+ let context = TestContext::new(None).await;
+ let team = context.insert_new_team(None).await.unwrap();
+
+ let json_string = payload_with_unparseable_flag(team.id).to_string();
+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)
+ .await
+ .unwrap();
+
+ let override_def = json!({
+ "id": 99,
+ "team_id": team.id,
+ "name": "Reconstructed deleted flag",
+ "key": "deleted-flag",
+ "filters": { "groups": [{"properties": [], "rollout_percentage": 100}] },
+ "deleted": false,
+ "active": true,
+ });
+ let overrides: HashMap<String, Value> =
+ HashMap::from([("deleted-flag".to_string(), override_def)]);
+
+ let result = fetch_and_filter(
+ &flag_service,
+ team.id,
+ &FlagsQueryParams::default(),
+ &HeaderMap::new(),
+ None,
+ None,
+ Some(&overrides),
+ )
+ .await
+ .unwrap();
+
+ assert!(
+ result.flags.iter().any(|f| f.key == "deleted-flag"),
+ "override must be added"
+ );
+ assert!(
+ result.evaluation_metadata.has_unparseable_flags,
+ "metadata recompute must keep the unparseable-flags marker"
+ );
+}
+
+/// The cached cohort list only holds cohorts the team's live flags reference,
+/// and the matcher treats a present list as complete. An override can name a
+/// cohort outside that list (a reconstructed deleted flag's cohort), so
+/// `fetch_and_filter` must drop the scoped list when an override was applied,
+/// which makes the matcher load the team's full cohort set instead. Without
+/// an override, the cached list passes through untouched.
+#[tokio::test]
+async fn test_fetch_and_filter_override_drops_scoped_cohort_list() {
+ let redis_client = setup_redis_client(None).await;
+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);
+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
+ let flag_service = FlagService::new(
+ redis_client.clone(),
+ reader.clone(),
+ team_hypercache_reader,
+ hypercache_reader,
+ Arc::new(FlagDefinitionsCache::disabled()),
+ NegativeCache::new(100, 300),
+ false,
+ );
+ let context = TestContext::new(None).await;
+ let team = context.insert_new_team(None).await.unwrap();
+
+ let flags = vec![mock!(FeatureFlag,
+ name: "Live flag".mock_into(),
+ id: 1,
+ key: "live-flag".mock_into(),
+ team_id: team.id,
+ filters: FlagFilters::default()
+ )];
+ let eval_metadata = EvaluationMetadata {
+ dependency_stages: vec![vec![1]],
+ flags_with_missing_deps: vec![],
+ transitive_deps: HashMap::from([(1, std::collections::HashSet::new())]),
+ ..Default::default()
+ };
+ // What Django writes when no live flag targets a cohort: an empty list,
+ // not a missing one.
+ let wrapper = HypercacheFlagsWrapper {
+ flags: flags.clone(),
+ evaluation_metadata: eval_metadata,
+ cohorts: Some(vec![]),
+ };
+ let json_string = serde_json::to_string(&wrapper).unwrap();
+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)
+ .await
+ .unwrap();
+
+ // Without an override the scoped list passes through.
+ let no_override = fetch_and_filter(
+ &flag_service,
+ team.id,
+ &FlagsQueryParams::default(),
+ &HeaderMap::new(),
+ None,
+ None,
+ None,
+ )
+ .await
+ .unwrap();
+ assert!(
+ no_override.cohorts.is_some(),
+ "without overrides the cached cohort list must pass through"
+ );
+
+ let override_def = json!({
+ "id": 2,
+ "team_id": team.id,
+ "name": "Reconstructed deleted flag",
+ "key": "deleted-flag",
+ "filters": { "groups": [{"properties": [], "rollout_percentage": 100}] },
+ "deleted": false,
+ "active": true,
+ });
+ let overrides: HashMap<String, Value> =
+ HashMap::from([("deleted-flag".to_string(), override_def)]);
+
+ let result = fetch_and_filter(
+ &flag_service,
+ team.id,
+ &FlagsQueryParams::default(),
+ &HeaderMap::new(),
+ None,
+ None,
+ Some(&overrides),
+ )
+ .await
+ .unwrap();
+
+ assert!(
+ result.flags.iter().any(|f| f.key == "deleted-flag"),
+ "override must be added"
+ );
+ assert!(
+ result.cohorts.is_none(),
+ "an applied override must drop the scoped cohort list so the matcher loads the full set"
+ );
+}
+
+/// A REPLACED flag's historical version can depend on flags its live version
+/// does not. The recompute must run for replacements too, or the cached
+/// `transitive_deps` would leave the dependency unevaluated and the condition
+/// would read as a silent non-match.
+#[tokio::test]
+async fn test_fetch_and_filter_replace_override_recomputes_metadata() {
+ let redis_client = setup_redis_client(None).await;
+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);
+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
+ let flag_service = FlagService::new(
+ redis_client.clone(),
+ reader.clone(),
+ team_hypercache_reader,
+ hypercache_reader,
+ Arc::new(FlagDefinitionsCache::disabled()),
+ NegativeCache::new(100, 300),
+ false,
+ );
+ let context = TestContext::new(None).await;
+ let team = context.insert_new_team(None).await.unwrap();
+
+ // Two independent live flags: the cached metadata has no edges.
+ let flags = vec![
+ mock!(FeatureFlag,
+ name: "Flag A".mock_into(),
+ id: 1,
+ key: "flag-a".mock_into(),
+ team_id: team.id,
+ filters: FlagFilters::default()
+ ),
+ mock!(FeatureFlag,
+ name: "Flag B".mock_into(),
+ id: 2,
+ key: "flag-b".mock_into(),
+ team_id: team.id,
+ filters: FlagFilters::default()
+ ),
+ ];
+ let eval_metadata = EvaluationMetadata {
+ dependency_stages: vec![vec![1, 2]],
+ flags_with_missing_deps: vec![],
+ transitive_deps: HashMap::from([
+ (1, std::collections::HashSet::new()),
+ (2, std::collections::HashSet::new()),
+ ]),
+ ..Default::default()
+ };
+ let wrapper = HypercacheFlagsWrapper {
+ flags: flags.clone(),
+ evaluation_metadata: eval_metadata,
+ cohorts: Some(vec![]),
+ };
+ let json_string = serde_json::to_string(&wrapper).unwrap();
+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)
+ .await
+ .unwrap();
+
+ // The historical version of flag-a depended on flag-b.
+ let override_def = json!({
+ "id": 1,
+ "team_id": team.id,
+ "name": "Flag A at the timestamp",
+ "key": "flag-a",
+ "filters": { "groups": [{
+ "properties": [{
+ "key": "2",
+ "type": "flag",
+ "value": "true",
+ "operator": "flag_evaluates_to",
+ }],
+ "rollout_percentage": 100,
+ }] },
+ "deleted": false,
+ "active": true,
+ });
+ let overrides: HashMap<String, Value> = HashMap::from([("flag-a".to_string(), override_def)]);
+
+ let result = fetch_and_filter(
+ &flag_service,
+ team.id,
+ &FlagsQueryParams::default(),
+ &HeaderMap::new(),
+ None,
+ None,
+ Some(&overrides),
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(
+ result.evaluation_metadata.transitive_deps.get(&1),
+ Some(&std::collections::HashSet::from([2])),
+ "replace-path override must recompute transitive deps (got {:?})",
+ result.evaluation_metadata.transitive_deps
+ );
+ assert!(
+ result.cohorts.is_none(),
+ "an applied override must drop the scoped cohort list"
+ );
+}
+
+/// An added flag can depend on a flag that is itself still deleted and so
+/// absent from the merged list. The metadata recompute must mark the added
+/// flag as missing a dependency (it then fails closed), and the live flag
+/// must stay staged.
+#[tokio::test]
+async fn test_fetch_and_filter_override_add_with_missing_dependency_fails_closed() {
+ let redis_client = setup_redis_client(None).await;
+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);
+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;
+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;
+ let flag_service = FlagService::new(
+ redis_client.clone(),
+ reader.clone(),
+ team_hypercache_reader,
+ hypercache_reader,
+ Arc::new(FlagDefinitionsCache::disabled()),
+ NegativeCache::new(100, 300),
+ false,
+ );
+ let context = TestContext::new(None).await;
+ let team = context.insert_new_team(None).await.unwrap();
+
+ let flags = vec![mock!(FeatureFlag,
+ name: "Live flag".mock_into(),
+ id: 1,
+ key: "live-flag".mock_into(),
+ team_id: team.id,
+ filters: FlagFilters::default()
+ )];
+ let eval_metadata = EvaluationMetadata {
+ dependency_stages: vec![vec![1]],
+ flags_with_missing_deps: vec![],
+ transitive_deps: HashMap::from([(1, std::collections::HashSet::new())]),
+ ..Default::default()
+ };
+ let wrapper = HypercacheFlagsWrapper {
+ flags: flags.clone(),
+ evaluation_metadata: eval_metadata,
+ cohorts: None,
+ };
+ let json_string = serde_json::to_string(&wrapper).unwrap();
+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)
+ .await
+ .unwrap();
+
+ // The added flag depends on flag id 99999, which is in no list (e.g. a
+ // dependency deleted in the same bulk delete).
+ let override_def = json!({
+ "id": 2,
+ "team_id": team.id,
+ "name": "Reconstructed deleted flag",
+ "key": "deleted-flag",
+ "filters": { "groups": [{
+ "properties": [{
+ "key": "99999",
+ "type": "flag",
+ "value": "true",
+ "operator": "flag_evaluates_to",
+ }],
+ "rollout_percentage": 100,
+ }] },
+ "deleted": false,
+ "active": true,
+ });
+ let overrides: HashMap<String, Value> =
+ HashMap::from([("deleted-flag".to_string(), override_def)]);
+
+ let result = fetch_and_filter(
+ &flag_service,
+ team.id,
+ &FlagsQueryParams::default(),
+ &HeaderMap::new(),
+ None,
+ None,
+ Some(&overrides),
+ )
+ .await
+ .unwrap();
+
+ let added = result
+ .flags
+ .iter()
+ .find(|f| f.key == "deleted-flag")
+ .expect("added flag must be in the list");
+ assert!(
+ result
+ .evaluation_metadata
+ .flags_with_missing_deps
+ .contains(&added.id),
+ "added flag with an unresolvable dependency must be marked missing-deps (got {:?})",
+ result.evaluation_metadata.flags_with_missing_deps
+ );
+ let staged: Vec<i32> = result
+ .evaluation_metadata
+ .dependency_stages
+ .iter()
+ .flatten()
+ .copied()
+ .collect();
+ assert!(
+ staged.contains(&1),
+ "live flag must stay staged (stages: {staged:?})"
+ );
+}
+
/// Two `fetch_and_filter` calls against a primed `FlagDefinitionsCache` must
/// share the same `Arc<[FeatureFlag]>` and `Arc<EvaluationMetadata>` — i.e.
/// `fetch_and_filter` must not deep-clone either field.