:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(flags): reconstruct deleted flags in historical test evaluation (#124511)
ruby childs0384d0b67ab0
diff --git a/products/feature_flags/backend/api/feature_flag.py b/products/feature_flags/backend/api/feature_flag.pyindex 807c80d6f125..faa5591f7dc0 100644--- a/products/feature_flags/backend/api/feature_flag.py+++ b/products/feature_flags/backend/api/feature_flag.py@@ -323,6 +323,13 @@ EARLY_EXIT_FLAG = "feature-flag-early-exit"ENFORCE_FEATURE_FLAG_WRITE_SCOPE_FLAG = "enforce-feature-flag-write-scope-cross-resource"ENCRYPTED_VERSION_HISTORY_UNAVAILABLE = "Version history is not available for flags with encrypted payloads."+ENCRYPTED_HISTORICAL_EVALUATION_UNAVAILABLE = (+ "Historical evaluation is not available for flags with encrypted payloads."+)+TOMBSTONED_KEY_REUSED_UNAVAILABLE = (+ "Historical evaluation is not available for this flag. Deleting it renamed its key, "+ "and a newer flag now uses the original key, so the original bucketing can't be reproduced."+)# What a config version 2 write may change, in request-field names; everything else stays unsupported# until the task that owns it lands. Disabling and soft-deleting are incident controls and need no admission.@@ -5968,6 +5975,17 @@ class FeatureFlagViewSet(evaluation_distinct_id = distinct_id if distinct_id and distinct_id in distinct_ids else sorted(distinct_ids)[0]person_properties: dict[str, Any] = {}+ # Mirror the versions endpoint: encrypted payloads are only served through+ # /remote_config, which handles decryption. The reconstructed override would+ # otherwise hand the flags service the stored ciphertext for a flag that the+ # team's flag list deliberately excludes, and the evaluation response would+ # echo it back as the payload.+ if timestamp and feature_flag.has_encrypted_payloads:+ return Response(+ {"error": ENCRYPTED_HISTORICAL_EVALUATION_UNAVAILABLE},+ status=status.HTTP_400_BAD_REQUEST,+ )+# Build person properties at timestamp if providedif timestamp:try:@@ -6016,6 +6034,16 @@ class FeatureFlagViewSet(timestamp=timestamp,team_id=self.team_id,)+ # `has_encrypted_payloads` is mutable: a flag downgraded to plaintext keeps+ # ciphertext in the activity log the reconstruction reads, so gate on the+ # reconstructed version's own state too (same rule as the versions endpoint).+ # Index rather than .get(): if a refactor ever drops the field from+ # reconstruction, this gate must fail loud, not quietly open.+ if reconstructed_flag_data["has_encrypted_payloads"]:+ return Response(+ {"error": ENCRYPTED_HISTORICAL_EVALUATION_UNAVAILABLE},+ status=status.HTTP_400_BAD_REQUEST,+ )evaluation_filters = reconstructed_flag_data.get("filters") or {}except VersionNotFound:@@ -6042,6 +6070,24 @@ class FeatureFlagViewSet(status=status.HTTP_500_INTERNAL_SERVER_ERROR,)+ # Bucketing hashes the flag key, and soft-deleting a flag linked to an+ # experiment renames its key to "<key>:deleted:<id>". A historical+ # evaluation must run under the original key, or users land in a+ # different hash bucket than they did historically and get a wrong+ # variant. If a live flag has reclaimed the original key, the flags+ # service would match that flag by key and answer with its current+ # state, so refuse instead.+ evaluation_key = feature_flag.key+ if timestamp and feature_flag.deleted:+ original_key = feature_flag.key_without_tombstone()+ if original_key != feature_flag.key:+ if FeatureFlag.objects.filter(team_id=feature_flag.team_id, key=original_key).exists():+ return Response(+ {"error": TOMBSTONED_KEY_REUSED_UNAVAILABLE},+ status=status.HTTP_400_BAD_REQUEST,+ )+ evaluation_key = original_key+# Evaluate the flagtry:# Get team API token for Rust service@@ -6062,13 +6108,21 @@ class FeatureFlagViewSet(flag_dict: dict[str, Any] = {k: reconstructed_flag_data[k] for k in RUST_FLAG_FIELDS if k in reconstructed_flag_data}- flag_dict.update({"id": feature_flag.id, "key": feature_flag.key, "team_id": feature_flag.team_id})+ flag_dict.update({"id": feature_flag.id, "key": evaluation_key, "team_id": feature_flag.team_id})++ # Reconstruction succeeded, so find_version_at_timestamp proved the+ # flag was not deleted at the timestamp. The reconstructed `deleted`+ # can still be True: a config-v1 bulk delete sets deleted=True+ # without a version bump and logs empty changes, so the walk-back+ # never undoes it. The flags service skips deleted flags, so the+ # override must ship deleted=False.+ flag_dict["deleted"] = False# Stringify datetime fields for JSON serialization.if flag_dict.get("created_at") and hasattr(flag_dict["created_at"], "isoformat"):flag_dict["created_at"] = flag_dict["created_at"].isoformat()- override_definitions = {feature_flag.key: flag_dict}+ override_definitions = {evaluation_key: flag_dict}except Exception as e:logger.exception("Failed to serialize flag for override")capture_exception(e)@@ -6091,7 +6145,7 @@ class FeatureFlagViewSet(detailed_analysis=True,person_properties=person_properties,only_use_override_person_properties=timestamp is not None,- flag_keys=[feature_flag.key],+ flag_keys=[evaluation_key],internal_request_token=internal_token,override_flags_definitions=override_definitions,# A pooled connection that the service closed fails once with a reset, so retry it.@@ -6101,7 +6155,7 @@ class FeatureFlagViewSet(# Extract the flag result from the Rust responseflags = rust_response.get("flags", {})- flag_result = flags.get(feature_flag.key)+ flag_result = flags.get(evaluation_key)# Initialize defaultscondition_index = None@@ -6126,6 +6180,26 @@ class FeatureFlagViewSet(reason_description = reason_data.get("description") if reason_data else Nonecondition_index = reason_data.get("condition_index") if reason_data else Nonepayload = metadata.get("payload") if metadata else None+ # A failed per-flag result also comes back without conditions.+ # Split it by cause: a dependency the definition cannot resolve+ # is deterministic, so a 400 tells the caller the request itself+ # cannot succeed. Anything else (timeouts, connection+ # exhaustion) is transient service trouble, so keep a logged+ # 5xx that retries and alerting can see.+ if timestamp and flag_result.get("failed"):+ if reason.startswith(("dependency_not_found", "dependency_cycle")):+ return Response(+ {"error": f"Historical evaluation failed: {reason_description or reason}."},+ status=status.HTTP_400_BAD_REQUEST,+ )+ logger.error(+ "Historical evaluation failed in the flags service",+ extra={"flag_key": feature_flag.key, "reason": reason},+ )+ return Response(+ {"error": f"Historical evaluation failed: {reason_description or reason}. Try again."},+ status=status.HTTP_502_BAD_GATEWAY,+ )# Extract conditions from flag result (only valid path per Rust FlagDetails contract).# When ``timestamp`` is set, missing conditions almost certainly means Rust# rejected the internal token and silently fell back to current-statediff --git a/products/feature_flags/backend/api/test/test_feature_flag.py b/products/feature_flags/backend/api/test/test_feature_flag.pyindex f309488e8dd7..fd8369c7ab9b 100644--- a/products/feature_flags/backend/api/test/test_feature_flag.py+++ b/products/feature_flags/backend/api/test/test_feature_flag.py@@ -14913,6 +14913,367 @@ class TestFeatureFlagTestEvaluation(APIBaseTest, ClickhouseTestMixin):# by the (reconstructed) flag's conditions.self.assertEqual(response.json()["person_properties"], {"email": "[email protected]"})+ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_before_single_soft_delete_sends_undeleted_override(+ self, mock_get_flags, _mock_build_props+ ):+ """A timestamp before a single soft delete must ship the reconstructed+ definition with deleted=False in the override payload. The flags service+ excludes deleted flags from the team's flag list, so this payload is the+ only vehicle for the historical definition — a deleted=True (or dropped)+ override surfaces as result=False, reason=flag_not_found in the UI+ Testing tab and the MCP tool."""+ filters = {"groups": [{"properties": [], "rollout_percentage": 100}]}+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="historical-deleted-flag",+ filters=filters,+ created_by=self.user,+ version=1,+ )+ create_person(team=self.team, distinct_ids=["test-user"])++ # Soft delete through the API so the real activity-logging path runs.+ delete_response = self.client.patch(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",+ {"deleted": True},+ format="json",+ )+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())+ flag.refresh_from_db()+ self.assertTrue(flag.deleted)++ mock_get_flags.return_value = {+ "flags": {+ "historical-deleted-flag": {+ "enabled": True,+ "variant": None,+ "reason": {"code": "condition_match", "condition_index": 0},+ "metadata": {},+ "conditions": [],+ }+ }+ }++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ sent = mock_get_flags.call_args.kwargs["override_flags_definitions"]["historical-deleted-flag"]+ self.assertIs(sent["deleted"], False, f"override payload must undelete the flag, got {sent!r}")+ self.assertEqual(sent["filters"], filters)+ self.assertEqual(sent["team_id"], self.team.pk)+ self.assertEqual(response.json()["result"], True)++ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_before_bulk_delete_sends_undeleted_override(self, mock_get_flags, _mock_build_props):+ """Bulk delete of a non-v2 flag writes deleted=True via .update() without a+ version bump and logs activity="deleted" with empty changes. Reconstruction+ starts from the current (deleted) row and the walk-back never undoes the+ deletion, so without a correction the override payload ships deleted=True+ and the flags service skips the flag. The endpoint must force+ deleted=False: a successful reconstruction already proves the flag existed+ and was not deleted at the timestamp."""+ filters = {"groups": [{"properties": [], "rollout_percentage": 100}]}+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="bulk-deleted-flag",+ filters=filters,+ created_by=self.user,+ version=1,+ )+ create_person(team=self.team, distinct_ids=["test-user"])++ delete_response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/bulk_delete/",+ {"ids": [flag.id]},+ format="json",+ )+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())+ flag.refresh_from_db()+ self.assertTrue(flag.deleted)++ mock_get_flags.return_value = {+ "flags": {+ "bulk-deleted-flag": {+ "enabled": True,+ "variant": None,+ "reason": {"code": "condition_match", "condition_index": 0},+ "metadata": {},+ "conditions": [],+ }+ }+ }++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ sent = mock_get_flags.call_args.kwargs["override_flags_definitions"]["bulk-deleted-flag"]+ self.assertIs(sent["deleted"], False, f"override payload must undelete the flag, got {sent!r}")+ self.assertEqual(sent["filters"], filters)+ self.assertEqual(response.json()["result"], True)++ # A timestamp after the deletion must still refuse: with deleted=False+ # forced onto the override, this 400 is the only thing stopping a+ # deleted flag from evaluating as live.+ after_response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": now().isoformat()},+ format="json",+ )+ self.assertEqual(after_response.status_code, status.HTTP_400_BAD_REQUEST, after_response.json())+ self.assertIn("did not exist", after_response.json()["error"])+ self.assertEqual(mock_get_flags.call_count, 1)++ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_with_timestamp_rejects_encrypted_payload_flags(self, mock_get_flags, mock_build_props):+ """The reconstructed override would carry the stored ciphertext for a flag the+ team's flag list deliberately excludes, so historical evaluation must refuse+ encrypted-payload flags the same way the versions endpoint does. The first+ gate fires before any ClickHouse or flags-service work."""+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="encrypted-flag",+ filters={+ "groups": [{"properties": [], "rollout_percentage": 100}],+ "payloads": {"true": "ciphertext"},+ },+ has_encrypted_payloads=True,+ created_by=self.user,+ version=1,+ )+ create_person(team=self.team, distinct_ids=["test-user"])++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST)+ self.assertIn("encrypted payloads", response.json()["error"])+ mock_build_props.assert_not_called()+ mock_get_flags.assert_not_called()++ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_rejects_flag_encrypted_at_timestamp(self, mock_get_flags, _mock_build_props):+ """The second encrypted gate: a flag that is plaintext now but had+ encrypted payloads at the timestamp must still be refused, or the+ reconstruction would ship the activity log's stored ciphertext to the+ flags service and echo it back as the payload."""+ flag = FeatureFlag.objects.create(+ team=self.team,+ created_by=self.user,+ key="downgraded-flag",+ filters={+ "groups": [{"properties": [], "rollout_percentage": 100}],+ "payloads": {"true": "ciphertext"},+ },+ is_remote_configuration=True,+ has_encrypted_payloads=True,+ version=1,+ )+ create_person(team=self.team, distinct_ids=["test-user"])++ # Downgrade through the API so the change is activity-logged.+ downgrade = self.client.patch(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",+ {+ "has_encrypted_payloads": False,+ "is_remote_configuration": False,+ "filters": {"groups": [{"properties": [], "rollout_percentage": 100}], "payloads": {}},+ },+ format="json",+ )+ self.assertEqual(downgrade.status_code, status.HTTP_200_OK, downgrade.json())++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())+ self.assertIn("encrypted payloads", response.json()["error"])+ mock_get_flags.assert_not_called()++ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_deleted_experiment_flag_uses_original_key(self, mock_get_flags, _mock_build_props):+ """Deleting a flag linked to an experiment renames its key to+ "<key>:deleted:<id>". Bucketing hashes the key, so the override must+ carry the original key or historical evaluation reports the wrong+ bucket (and on a multivariate split, the wrong variant)."""+ filters = {"groups": [{"properties": [], "rollout_percentage": 50}]}+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="exp-flag",+ filters=filters,+ created_by=self.user,+ version=1,+ )+ create_experiment(team_id=self.team.id, feature_flag_id=flag.id, created_by_id=self.user.id, deleted=True)+ create_person(team=self.team, distinct_ids=["test-user"])++ delete_response = self.client.patch(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",+ {"deleted": True},+ format="json",+ )+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())+ flag.refresh_from_db()+ self.assertEqual(flag.key, f"exp-flag:deleted:{flag.id}")++ mock_get_flags.return_value = {+ "flags": {+ "exp-flag": {+ "enabled": True,+ "variant": None,+ "reason": {"code": "condition_match", "condition_index": 0},+ "metadata": {},+ "conditions": [],+ }+ }+ }++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_200_OK, response.json())+ overrides = mock_get_flags.call_args.kwargs["override_flags_definitions"]+ self.assertEqual(list(overrides.keys()), ["exp-flag"])+ self.assertEqual(overrides["exp-flag"]["key"], "exp-flag")+ self.assertEqual(mock_get_flags.call_args.kwargs["flag_keys"], ["exp-flag"])+ self.assertEqual(response.json()["result"], True)++ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_tombstoned_key_reclaimed_by_live_flag_refuses(self, mock_get_flags, _mock_build_props):+ """When a live flag has reclaimed a deleted flag's original key, the+ flags service would match the live flag by key and answer with its+ current state, which would read as a historical result. Refuse instead."""+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="reclaimed-flag",+ filters={"groups": [{"properties": [], "rollout_percentage": 50}]},+ created_by=self.user,+ version=1,+ )+ create_experiment(team_id=self.team.id, feature_flag_id=flag.id, created_by_id=self.user.id, deleted=True)+ create_person(team=self.team, distinct_ids=["test-user"])++ delete_response = self.client.patch(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/",+ {"deleted": True},+ format="json",+ )+ self.assertEqual(delete_response.status_code, status.HTTP_200_OK, delete_response.json())+ FeatureFlag.objects.create(team=self.team, key="reclaimed-flag", created_by=self.user, version=1)++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())+ self.assertIn("original key", response.json()["error"])+ mock_get_flags.assert_not_called()++ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_failed_flag_result_returns_evaluation_error(self, mock_get_flags, _mock_build_props):+ """A failed per-flag result (e.g. a cohort the service could not+ resolve) must surface as an evaluation error, not as the 502 that+ blames INTERNAL_REQUEST_TOKEN configuration."""+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="failing-flag",+ filters={"groups": [{"properties": [], "rollout_percentage": 100}]},+ created_by=self.user,+ version=1,+ )+ create_person(team=self.team, distinct_ids=["test-user"])++ mock_get_flags.return_value = {+ "flags": {+ "failing-flag": {+ "enabled": False,+ "failed": True,+ "variant": None,+ "reason": {"code": "dependency_not_found_cohort", "description": "Cohort dependency not found"},+ "metadata": {},+ }+ }+ }++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST, response.json())+ self.assertIn("Cohort dependency not found", response.json()["error"])++ @patch("products.feature_flags.backend.api.feature_flag.build_person_properties_at_time", return_value={})+ @patch("products.feature_flags.backend.api.feature_flag.get_flags_from_service")+ @override_settings(INTERNAL_REQUEST_TOKEN="test-token")+ def test_test_evaluation_transient_failure_returns_502(self, mock_get_flags, _mock_build_props):+ """A transient service failure (timeout, connection exhaustion) is not a+ property of the request, so it keeps a 5xx that retries and alerting+ can see, unlike the deterministic dependency 400 above."""+ flag = FeatureFlag.objects.create(+ team=self.team,+ key="transient-failing-flag",+ filters={"groups": [{"properties": [], "rollout_percentage": 100}]},+ created_by=self.user,+ version=1,+ )+ create_person(team=self.team, distinct_ids=["test-user"])++ mock_get_flags.return_value = {+ "flags": {+ "transient-failing-flag": {+ "enabled": False,+ "failed": True,+ "variant": None,+ "reason": {"code": "group_mapping_retry", "description": "Failed to fetch group mapping"},+ "metadata": {},+ }+ }+ }++ response = self.client.post(+ f"/api/projects/{self.team.pk}/feature_flags/{flag.id}/test_evaluation/",+ {"distinct_id": "test-user", "timestamp": flag.created_at.isoformat()},+ format="json",+ )++ self.assertEqual(response.status_code, status.HTTP_502_BAD_GATEWAY, response.json())+ self.assertIn("Failed to fetch group mapping", response.json()["error"])+def test_test_evaluation_distinct_id_person_id_conflict(self):"""Test validation error when both distinct_id and person_id are provided."""flag = FeatureFlag.objects.create(team=self.team, key="test-flag")diff --git a/products/feature_flags/backend/test/test_version_history.py b/products/feature_flags/backend/test/test_version_history.pyindex b56cf489faa2..5ccb88e9709a 100644--- a/products/feature_flags/backend/test/test_version_history.py+++ b/products/feature_flags/backend/test/test_version_history.py@@ -467,6 +467,111 @@ class TestTimestampBasedReconstruction(TestReconstructFlagAtVersion):version = find_version_at_timestamp(flag, current_timestamp, self.team.id)assert version is None+ def test_find_version_at_timestamp_deleted_flag_with_later_versioned_edit(self):+ """A versioned edit logged after the deletion must not mask it: the flag+ was still deleted at that time, so the version lookup returns None."""+ flag = self._create_flag(name="V1", version=1)++ log_activity(+ organization_id=self.organization.id,+ team_id=self.team.id,+ user=self.user,+ was_impersonated=False,+ item_id=flag.id,+ scope="FeatureFlag",+ activity="deleted",+ detail=Detail(+ changes=[Change(type="FeatureFlag", action="changed", field="version", before=1, after=2)],+ name=flag.key,+ ),+ )+ # A write to the deleted flag's row that logs a versioned "updated" entry.+ log_activity(+ organization_id=self.organization.id,+ team_id=self.team.id,+ user=self.user,+ was_impersonated=False,+ item_id=flag.id,+ scope="FeatureFlag",+ activity="updated",+ detail=Detail(+ changes=[Change(type="FeatureFlag", action="changed", field="version", before=2, after=3)],+ name=flag.key,+ ),+ )++ version = find_version_at_timestamp(flag, flag.created_at + timedelta(hours=1), self.team.id)+ assert version is None++ def test_find_version_at_timestamp_legacy_deletion_logged_as_updated(self):+ """Before #41043 (2025-11-19), a soft delete through PATCH was logged+ as activity="updated" with a `deleted` False-to-True change. Those+ entries still decide existence: a timestamp after one must return None,+ or the endpoint's forced deleted=False revives the flag."""+ flag = self._create_flag(name="V1", version=1)++ log_activity(+ organization_id=self.organization.id,+ team_id=self.team.id,+ user=self.user,+ was_impersonated=False,+ item_id=flag.id,+ scope="FeatureFlag",+ activity="updated",+ detail=Detail(+ changes=[+ Change(type="FeatureFlag", action="changed", field="deleted", before=False, after=True),+ Change(type="FeatureFlag", action="changed", field="version", before=1, after=2),+ ],+ name=flag.key,+ ),+ )++ version = find_version_at_timestamp(flag, flag.created_at + timedelta(hours=1), self.team.id)+ assert version is None++ def test_find_version_at_timestamp_legacy_restore_logged_as_updated(self):+ """The legacy counterpart of a restore: an activity="updated" entry+ with a `deleted` True-to-False change confirms the flag was alive, so+ an older legacy deletion must not hide the version."""+ flag = self._create_flag(name="V1", version=1)++ log_activity(+ organization_id=self.organization.id,+ team_id=self.team.id,+ user=self.user,+ was_impersonated=False,+ item_id=flag.id,+ scope="FeatureFlag",+ activity="updated",+ detail=Detail(+ changes=[+ Change(type="FeatureFlag", action="changed", field="deleted", before=False, after=True),+ Change(type="FeatureFlag", action="changed", field="version", before=1, after=2),+ ],+ name=flag.key,+ ),+ )+ log_activity(+ organization_id=self.organization.id,+ team_id=self.team.id,+ user=self.user,+ was_impersonated=False,+ item_id=flag.id,+ scope="FeatureFlag",+ activity="updated",+ detail=Detail(+ changes=[+ Change(type="FeatureFlag", action="changed", field="deleted", before=True, after=False),+ Change(type="FeatureFlag", action="changed", field="version", before=2, after=3),+ ],+ name=flag.key,+ ),+ )++ version = find_version_at_timestamp(flag, flag.created_at + timedelta(hours=1), self.team.id)+ assert version == 3+def test_find_version_at_timestamp_soft_deleted_then_restored(self):"""Test that restored flags return proper version information."""flag = self._create_flag(name="V1", version=1)diff --git a/products/feature_flags/backend/version_history.py b/products/feature_flags/backend/version_history.pyindex 4adbe1d616fa..6fe4709af8c9 100644--- a/products/feature_flags/backend/version_history.py+++ b/products/feature_flags/backend/version_history.py@@ -1,6 +1,7 @@from datetime import datetimefrom typing import Any+from django.db.models import Qfrom django.utils import timezonefrom posthog.models.activity_logging.activity_log import ActivityLog, common_field_exclusions, field_exclusions@@ -209,35 +210,51 @@ def find_version_at_timestamp(flag: FeatureFlag, timestamp: datetime, team_id: iif timestamp < flag.created_at:return None- # Get activity log entries at or before the timestamp, newest first- entries = (- ActivityLog.objects.filter(- team_id=team_id,- scope="FeatureFlag",- item_id=str(flag.id),- activity__in=["updated", "deleted", "restored"],- created_at__lte=timestamp,- )- .order_by("-created_at", "-id")- .values_list("activity", "detail")[:MAX_HISTORY_ENTRIES]- .iterator()+ history = ActivityLog.objects.filter(+ team_id=team_id,+ scope="FeatureFlag",+ item_id=str(flag.id),+ created_at__lte=timestamp,+ ).order_by("-created_at", "-id")++ # Deletion state and version are resolved independently. The most recent+ # entry that changed deletion state at or before the timestamp decides+ # existence, while a versioned "updated" entry does not, since a deleted+ # flag's row can still receive versioned writes (and bulk deletes log+ # "deleted" with empty changes). Deletion state comes from the activity+ # name AND from a `deleted` field change: before #41043 (2025-11-19), soft+ # deletes and restores through PATCH were logged as "updated" with a+ # `deleted` change, and those legacy entries still decide existence.+ lifecycle_entry = (+ history.filter(Q(activity__in=["deleted", "restored"]) | Q(detail__changes__contains=[{"field": "deleted"}]))+ .values_list("activity", "detail")+ .first())-- for activity, detail in entries:+ if lifecycle_entry is not None:+ activity, detail = lifecycle_entrychanges = (detail or {}).get("changes") or []- version_after = _get_version_after(changes)-- # Track deletion/restoration regardless of whether this entry carried a- # version change. Bulk deletes are logged with empty changes.- if activity == "deleted":+ deleted_after = next(+ (change.get("after") for change in changes if change.get("field") == "deleted"),+ None,+ )+ marks_deleted = activity == "deleted" or bool(deleted_after)+ if marks_deleted:return None+ # The most recent entry with a version change decides the version.+ details = (+ history.filter(activity__in=["updated", "deleted", "restored"])+ .values_list("detail", flat=True)[:MAX_HISTORY_ENTRIES]+ .iterator()+ )+ for detail in details:+ version_after = _get_version_after((detail or {}).get("changes") or [])if version_after is not None:return version_after- # No activity log entries found. If timestamp is after the flag's last update,- # return the current version. The scan has authoritative deletion state,- # or we wouldn't reach here.+ # No versioned entries found. If timestamp is after the flag's last update,+ # return the current version. The lifecycle lookup above has authoritative+ # deletion state, or we wouldn't reach here.if timestamp >= (flag.updated_at or flag.created_at):return flag.versiondiff --git a/rust/feature-flags/src/handler/flags.rs b/rust/feature-flags/src/handler/flags.rsindex 6900fe6268fe..bcd9b5850814 100644--- a/rust/feature-flags/src/handler/flags.rs+++ b/rust/feature-flags/src/handler/flags.rs@@ -4,9 +4,10 @@ use crate::{types::{FlagsQueryParams, FlagsResponse},},flags::{+ cache_builder::compute_flag_dependencies_or_single_stage,feature_flag_list::PreparedFlags,flag_analytics::SURVEY_TARGETING_FLAG_PREFIX,- flag_models::{FeatureFlag, FeatureFlagList},+ flag_models::{EvaluationMetadata, FeatureFlag, FeatureFlagList},flag_service::FlagService,},utils::user_agent::{RuntimeType, UserAgentInfo},@@ -141,23 +142,25 @@ fn collect_excluded_by_runtime(/// are dropped with a warning — a caller must not be able to swap one flag's/// identity for another (potentially cross-team) by submitting a forged/// override payload.+///+/// An override whose key is absent from ``flags`` is ADDED rather than+/// dropped: historical test evaluation reconstructs soft-deleted flags, and+/// every source of the team's flag list (hypercache and PG) excludes deleted+/// flags, so the reconstructed definition can only arrive this way. (The list+/// also omits disabled flags that no other flag references, so those take the+/// add path too when evaluated historically.) The added definition must carry+/// the request's own ``team_id``: overrides only reach this path through+/// internal-token requests, but the guard keeps a forged payload from+/// evaluating another team's flag definition.fn apply_flag_overrides(- flags: &mut [FeatureFlag],+ flags: &mut Vec<FeatureFlag>,override_defs: &HashMap<String, Value>,+ request_team_id: TeamId,) -> Vec<String> {let mut overridden_keys = Vec::new();tracing::debug!("Processing {} override definitions", override_defs.len());for (flag_key, override_def) in override_defs {tracing::debug!("Processing override for flag: {}", flag_key);- let Some(flag) = flags.iter_mut().find(|f| &f.key == flag_key) else {- tracing::warn!("Flag not found for override: {}", flag_key);- continue;- };- tracing::trace!(- "Found flag to override: {}, current filters: {:?}",- flag_key,- flag.filters- );let override_flag = match serde_json::from_value::<FeatureFlag>(override_def.clone()) {Ok(parsed) => parsed,Err(e) => {@@ -170,32 +173,49 @@ fn apply_flag_overrides(continue;}};- // Identity guard: a caller must not be able to use the override to- // swap in another flag's identity. The override is keyed by flag_key- // alone, so we reject payloads whose embedded key/id/team_id disagree- // with the flag we resolved from the team's flag list.- if override_flag.key != *flag_key- || override_flag.id != flag.id- || override_flag.team_id != flag.team_id- {+ // Identity guards shared by both paths: the override map is keyed by+ // flag_key alone, so the payload's embedded key must match it, and the+ // payload must belong to the requesting team (every flag in `flags`+ // does, since both list sources filter by team).+ if override_flag.key != *flag_key || override_flag.team_id != request_team_id {tracing::warn!(- "Override identity mismatch for flag {}: skipping (override key={:?}, id={}, team_id={}; expected key={:?}, id={}, team_id={})",+ "Override identity mismatch for flag {}: skipping (override key={:?}, team_id={}; request team_id={})",flag_key,override_flag.key,- override_flag.id,override_flag.team_id,- flag.key,- flag.id,- flag.team_id+ request_team_id);continue;}- tracing::trace!(- "Successfully parsed override flag: {}, new filters: {:?}",- flag_key,- override_flag.filters- );- *flag = override_flag;+ match flags.iter().position(|f| &f.key == flag_key) {+ Some(index) => {+ // A payload must not swap in another flag's identity.+ if override_flag.id != flags[index].id {+ tracing::warn!(+ "Override for flag {} has id {}, expected {}: skipping",+ flag_key,+ override_flag.id,+ flags[index].id+ );+ continue;+ }+ flags[index] = override_flag;+ }+ None => {+ // The key is not in the team's list, so this is a reconstructed+ // deleted flag: add it. Evaluation stages are keyed by id, so+ // the added flag must not reuse an existing flag's id.+ if flags.iter().any(|f| f.id == override_flag.id) {+ tracing::warn!(+ "Override for flag {} reuses id {} of an existing flag: skipping",+ flag_key,+ override_flag.id+ );+ continue;+ }+ flags.push(override_flag);+ }+ }overridden_keys.push(flag_key.clone());}overridden_keys@@ -219,24 +239,54 @@ pub async fn fetch_and_filter(// Apply override flag definitions if provided. Overrides require a clone// because `prepared.flags` is Arc-backed and shared across requests.- let overridden_flags: Option<PreparedFlags> = match override_flags_definitions {- Some(override_defs) => {- let mut flags_vec: Vec<FeatureFlag> = prepared.flags.iter().cloned().collect();- let overridden_keys = apply_flag_overrides(&mut flags_vec, override_defs);- if !overridden_keys.is_empty() {- with_canonical_log(|log| {- log.flags_overridden = Some(overridden_keys);- });+ // `overridden` is Some only when an override actually applied; a request+ // whose overrides were all rejected falls back to the cached list. The+ // cached evaluation metadata was computed from the live definitions: an+ // ADDED flag (a reconstructed soft-deleted flag) sits in no dependency+ // stage, so it would never evaluate and the staging math would miscount+ // it as a cycle participant, while a REPLACED flag's historical version+ // can depend on flags its live version does not, so the cached+ // `transitive_deps` would leave those dependencies unevaluated.+ // Recomputing from the merged list keeps metadata and flag list+ // consistent. A dependency of an added flag that is itself still deleted+ // stays absent from the list, so the recompute marks the added flag as+ // missing a dependency and it fails closed, same as any unresolvable+ // dependency.+ let overridden: Option<(PreparedFlags, Arc<EvaluationMetadata>)> =+ match override_flags_definitions {+ Some(override_defs) => {+ let mut flags_vec: Vec<FeatureFlag> = prepared.flags.iter().cloned().collect();+ let overridden_keys = apply_flag_overrides(&mut flags_vec, override_defs, team_id);+ if overridden_keys.is_empty() {+ None+ } else {+ with_canonical_log(|log| {+ log.flags_overridden = Some(overridden_keys);+ });+ // `has_unparseable_flags` is read-time state (set when the+ // cache read skipped or omitted a flag), not derivable from+ // the flag list, so the recompute must carry it over or a+ // partial evaluation would stop reporting+ // errorsWhileComputingFlags.+ let metadata = Arc::new(EvaluationMetadata {+ has_unparseable_flags: prepared.evaluation_metadata.has_unparseable_flags,+ ..compute_flag_dependencies_or_single_stage(team_id, &flags_vec)+ });+ Some((PreparedFlags::seal(flags_vec), metadata))+ }}- Some(PreparedFlags::seal(flags_vec))- }- None => None,+ None => None,+ };+ let overrides_applied = overridden.is_some();+ let (prepared_flags, evaluation_metadata) = match overridden {+ Some((flags, metadata)) => (flags, metadata),+ None => (+ PreparedFlags::from_arc(Arc::clone(prepared.flags.as_arc())),+ Arc::clone(&prepared.evaluation_metadata),+ ),};- let flags: &[FeatureFlag] = match &overridden_flags {- Some(p) => p,- None => &prepared.flags,- };+ let flags: &[FeatureFlag] = &prepared_flags;// Build the filtered-out set: user-disabled, deleted, survey filter, runtime/tag mismatches.// This is the single source of truth for "should this flag be skipped during evaluation."@@ -274,13 +324,20 @@ pub async fn fetch_and_filter(// of refcount bumps rather than a deep copy of the flag slice, the// `EvaluationMetadata` map, or the cohort vec.let flag_list = FeatureFlagList {- flags: match overridden_flags {- Some(p) => p,- None => PreparedFlags::from_arc(Arc::clone(prepared.flags.as_arc())),- },+ flags: prepared_flags,filtered_out_flag_ids,- evaluation_metadata: Arc::clone(&prepared.evaluation_metadata),- cohorts: prepared.cohorts.as_ref().map(Arc::clone),+ evaluation_metadata,+ // The cached cohort list only holds cohorts the team's live flags+ // reference. An override can name others (a reconstructed deleted+ // flag's cohort, or one an older version of a live flag targeted), and+ // the matcher treats a present list as complete, so a missing cohort+ // fails evaluation with DependencyNotFound. Dropping the list makes+ // the matcher load the team's full cohort set instead.+ cohorts: if overrides_applied {+ None+ } else {+ prepared.cohorts.as_ref().map(Arc::clone)+ },};Ok(flag_list)}@@ -979,7 +1036,7 @@ mod tests {let mut overrides = HashMap::new();overrides.insert("my-flag".to_string(), override_payload(1, 100, "my-flag"));- let applied = apply_flag_overrides(&mut flags, &overrides);+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);assert_eq!(applied, vec!["my-flag".to_string()]);assert_eq!(flags[0].filters.groups[0].rollout_percentage, Some(50.0));@@ -993,7 +1050,7 @@ mod tests {// caller from swapping in another team's identity.overrides.insert("my-flag".to_string(), override_payload(1, 999, "my-flag"));- let applied = apply_flag_overrides(&mut flags, &overrides);+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);assert!(applied.is_empty(), "mismatched team_id must not be applied");assert_eq!(flags[0].team_id, 100, "original team_id must be preserved");@@ -1005,7 +1062,7 @@ mod tests {let mut overrides = HashMap::new();overrides.insert("my-flag".to_string(), override_payload(999, 100, "my-flag"));- let applied = apply_flag_overrides(&mut flags, &overrides);+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);assert!(applied.is_empty(), "mismatched id must not be applied");assert_eq!(flags[0].id, 1, "original id must be preserved");@@ -1022,22 +1079,107 @@ mod tests {override_payload(1, 100, "other-flag"),);- let applied = apply_flag_overrides(&mut flags, &overrides);+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);assert!(applied.is_empty(), "mismatched key must not be applied");}#[test]- fn test_apply_flag_overrides_skips_unknown_keys() {- let mut flags = vec![flag_with_team(1, 100, "my-flag")];+ fn test_apply_flag_overrides_adds_flag_missing_from_team_list() {+ // Historical test evaluation reconstructs a soft-deleted flag and ships it+ // as an override. Every source of the team's flag list (cache and PG) excludes+ // deleted flags, so the key is absent from `flags` and the override must be+ // ADDED, not dropped, or /test_evaluation reports flag_not_found for+ // timestamps at which the flag existed.+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];+ let mut overrides = HashMap::new();+ overrides.insert(+ "deleted-flag".to_string(),+ override_payload(2, 100, "deleted-flag"),+ );++ let applied = apply_flag_overrides(&mut flags, &overrides, 100);++ assert_eq!(+ applied,+ vec!["deleted-flag".to_string()],+ "override for a key absent from the team's flag list must be applied"+ );+ assert_eq!(+ flags.len(),+ 2,+ "the override definition must be added to the flag list"+ );+ let added = flags+ .iter()+ .find(|f| f.key == "deleted-flag")+ .expect("added flag must be present");+ assert_eq!(added.id, 2);+ assert_eq!(+ added.team_id, 100,+ "an added override must belong to the same team as the request"+ );+ assert_eq!(added.filters.groups[0].rollout_percentage, Some(50.0));+ }++ #[test]+ fn test_apply_flag_overrides_rejects_added_flag_for_other_team() {+ // An override for a key absent from the list may only be added when its+ // team_id matches the requesting team, otherwise a forged payload could+ // evaluate another team's flag definition.+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];let mut overrides = HashMap::new();overrides.insert(- "not-in-list".to_string(),- override_payload(1, 100, "not-in-list"),+ "deleted-flag".to_string(),+ override_payload(2, 999, "deleted-flag"),);- let applied = apply_flag_overrides(&mut flags, &overrides);+ let applied = apply_flag_overrides(&mut flags, &overrides, 100);- assert!(applied.is_empty());+ assert!(applied.is_empty(), "cross-team override must not be added");+ assert_eq!(flags.len(), 1, "flag list must be unchanged");+ }++ #[test]+ fn test_apply_flag_overrides_rejects_added_flag_with_duplicate_id() {+ // An added override must not reuse an existing flag's id: the+ // evaluation stages are keyed by id, so a duplicate would corrupt+ // staging and filtering.+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];+ let mut overrides = HashMap::new();+ overrides.insert(+ "deleted-flag".to_string(),+ override_payload(1, 100, "deleted-flag"),+ );++ let applied = apply_flag_overrides(&mut flags, &overrides, 100);++ assert!(+ applied.is_empty(),+ "duplicate-id override must not be added"+ );+ assert_eq!(flags.len(), 1, "flag list must be unchanged");+ }++ #[test]+ fn test_apply_flag_overrides_rejects_added_flag_with_mismatched_payload_key() {+ // An added override's payload key must equal the map key it arrived+ // under. Without this guard, a payload claiming an existing flag's key+ // would be pushed next to the real flag, and the key-indexed response+ // would silently shadow one of them.+ let mut flags = vec![flag_with_team(1, 100, "live-flag")];+ let mut overrides = HashMap::new();+ overrides.insert(+ "deleted-flag".to_string(),+ override_payload(2, 100, "live-flag"),+ );++ let applied = apply_flag_overrides(&mut flags, &overrides, 100);++ assert!(+ applied.is_empty(),+ "override whose payload key differs from its map key must not be added"+ );+ assert_eq!(flags.len(), 1, "flag list must be unchanged");}}diff --git a/rust/feature-flags/src/handler/tests.rs b/rust/feature-flags/src/handler/tests.rsindex 379c874ee568..0d4c67d23a0f 100644--- a/rust/feature-flags/src/handler/tests.rs+++ b/rust/feature-flags/src/handler/tests.rs@@ -39,7 +39,8 @@ use crate::{test_utils::{flag_list_with_metadata_and_filter, insert_flags_for_team_in_redis,mock_group_type_cache, setup_hypercache_reader, setup_pg_reader_client,- setup_pg_writer_client, setup_redis_client, setup_team_hypercache_reader, TestContext,+ setup_pg_writer_client, setup_redis_client, setup_team_hypercache_reader,+ write_flags_wire_json_to_redis, TestContext,},},};@@ -1499,6 +1500,477 @@ async fn test_fetch_and_filter_preserves_evaluation_metadata() {assert!(ctx.transitive_deps.contains_key(&2));}+/// Historical test evaluation ships a reconstructed soft-deleted flag as an+/// override. The team's cached flag list excludes deleted flags, so the+/// override's key is absent from it: `fetch_and_filter` must add the override+/// to the list, leave it un-filtered, and stage it in the evaluation metadata+/// (a flag absent from `dependency_stages` is never evaluated and would be+/// miscounted as a cycle participant).+#[tokio::test]+async fn test_fetch_and_filter_override_adds_flag_missing_from_team_list() {+ let redis_client = setup_redis_client(None).await;+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;+ let flag_service = FlagService::new(+ redis_client.clone(),+ reader.clone(),+ team_hypercache_reader,+ hypercache_reader,+ Arc::new(FlagDefinitionsCache::disabled()),+ NegativeCache::new(100, 300),+ false,+ );+ let context = TestContext::new(None).await;+ let team = context.insert_new_team(None).await.unwrap();++ // The team's list carries only the live flag; the soft-deleted flag (id 2)+ // is excluded at cache-write time, exactly like production.+ let flags = vec![mock!(FeatureFlag,+ name: "Live flag".mock_into(),+ id: 1,+ key: "live-flag".mock_into(),+ team_id: team.id,+ filters: FlagFilters::default()+ )];+ let eval_metadata = EvaluationMetadata {+ dependency_stages: vec![vec![1]],+ flags_with_missing_deps: vec![],+ transitive_deps: HashMap::from([(1, std::collections::HashSet::new())]),+ ..Default::default()+ };+ let wrapper = HypercacheFlagsWrapper {+ flags: flags.clone(),+ evaluation_metadata: eval_metadata,+ cohorts: None,+ };+ let json_string = serde_json::to_string(&wrapper).unwrap();+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)+ .await+ .unwrap();++ let override_def = json!({+ "id": 2,+ "team_id": team.id,+ "name": "Reconstructed deleted flag",+ "key": "deleted-flag",+ "filters": { "groups": [{"properties": [], "rollout_percentage": 100}] },+ "deleted": false,+ "active": true,+ });+ let overrides: HashMap<String, Value> =+ HashMap::from([("deleted-flag".to_string(), override_def)]);++ let query_params = FlagsQueryParams::default();+ let result = fetch_and_filter(+ &flag_service,+ team.id,+ &query_params,+ &HeaderMap::new(),+ None,+ None,+ Some(&overrides),+ )+ .await+ .unwrap();++ assert_eq!(+ result.flags.len(),+ 2,+ "override for a key absent from the team's list must be added, not dropped"+ );+ let added = result+ .flags+ .iter()+ .find(|f| f.key == "deleted-flag")+ .expect("added flag must be in the list");+ assert!(+ !result.filtered_out_flag_ids.contains(&added.id),+ "added flag must not be filtered out"+ );+ let staged: Vec<i32> = result+ .evaluation_metadata+ .dependency_stages+ .iter()+ .flatten()+ .copied()+ .collect();+ assert!(+ staged.contains(&added.id),+ "added flag must appear in a dependency stage or it is never evaluated (stages: {staged:?})"+ );+}++/// When an override adds a flag, `fetch_and_filter` recomputes the evaluation+/// metadata. The recompute must keep the read-time `has_unparseable_flags`+/// marker, or a partial evaluation would stop reporting+/// `errorsWhileComputingFlags`.+#[tokio::test]+async fn test_fetch_and_filter_override_add_preserves_unparseable_marker() {+ use crate::flags::test_helpers::payload_with_unparseable_flag;++ let redis_client = setup_redis_client(None).await;+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;+ let flag_service = FlagService::new(+ redis_client.clone(),+ reader.clone(),+ team_hypercache_reader,+ hypercache_reader,+ Arc::new(FlagDefinitionsCache::disabled()),+ NegativeCache::new(100, 300),+ false,+ );+ let context = TestContext::new(None).await;+ let team = context.insert_new_team(None).await.unwrap();++ let json_string = payload_with_unparseable_flag(team.id).to_string();+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)+ .await+ .unwrap();++ let override_def = json!({+ "id": 99,+ "team_id": team.id,+ "name": "Reconstructed deleted flag",+ "key": "deleted-flag",+ "filters": { "groups": [{"properties": [], "rollout_percentage": 100}] },+ "deleted": false,+ "active": true,+ });+ let overrides: HashMap<String, Value> =+ HashMap::from([("deleted-flag".to_string(), override_def)]);++ let result = fetch_and_filter(+ &flag_service,+ team.id,+ &FlagsQueryParams::default(),+ &HeaderMap::new(),+ None,+ None,+ Some(&overrides),+ )+ .await+ .unwrap();++ assert!(+ result.flags.iter().any(|f| f.key == "deleted-flag"),+ "override must be added"+ );+ assert!(+ result.evaluation_metadata.has_unparseable_flags,+ "metadata recompute must keep the unparseable-flags marker"+ );+}++/// The cached cohort list only holds cohorts the team's live flags reference,+/// and the matcher treats a present list as complete. An override can name a+/// cohort outside that list (a reconstructed deleted flag's cohort), so+/// `fetch_and_filter` must drop the scoped list when an override was applied,+/// which makes the matcher load the team's full cohort set instead. Without+/// an override, the cached list passes through untouched.+#[tokio::test]+async fn test_fetch_and_filter_override_drops_scoped_cohort_list() {+ let redis_client = setup_redis_client(None).await;+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;+ let flag_service = FlagService::new(+ redis_client.clone(),+ reader.clone(),+ team_hypercache_reader,+ hypercache_reader,+ Arc::new(FlagDefinitionsCache::disabled()),+ NegativeCache::new(100, 300),+ false,+ );+ let context = TestContext::new(None).await;+ let team = context.insert_new_team(None).await.unwrap();++ let flags = vec![mock!(FeatureFlag,+ name: "Live flag".mock_into(),+ id: 1,+ key: "live-flag".mock_into(),+ team_id: team.id,+ filters: FlagFilters::default()+ )];+ let eval_metadata = EvaluationMetadata {+ dependency_stages: vec![vec![1]],+ flags_with_missing_deps: vec![],+ transitive_deps: HashMap::from([(1, std::collections::HashSet::new())]),+ ..Default::default()+ };+ // What Django writes when no live flag targets a cohort: an empty list,+ // not a missing one.+ let wrapper = HypercacheFlagsWrapper {+ flags: flags.clone(),+ evaluation_metadata: eval_metadata,+ cohorts: Some(vec![]),+ };+ let json_string = serde_json::to_string(&wrapper).unwrap();+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)+ .await+ .unwrap();++ // Without an override the scoped list passes through.+ let no_override = fetch_and_filter(+ &flag_service,+ team.id,+ &FlagsQueryParams::default(),+ &HeaderMap::new(),+ None,+ None,+ None,+ )+ .await+ .unwrap();+ assert!(+ no_override.cohorts.is_some(),+ "without overrides the cached cohort list must pass through"+ );++ let override_def = json!({+ "id": 2,+ "team_id": team.id,+ "name": "Reconstructed deleted flag",+ "key": "deleted-flag",+ "filters": { "groups": [{"properties": [], "rollout_percentage": 100}] },+ "deleted": false,+ "active": true,+ });+ let overrides: HashMap<String, Value> =+ HashMap::from([("deleted-flag".to_string(), override_def)]);++ let result = fetch_and_filter(+ &flag_service,+ team.id,+ &FlagsQueryParams::default(),+ &HeaderMap::new(),+ None,+ None,+ Some(&overrides),+ )+ .await+ .unwrap();++ assert!(+ result.flags.iter().any(|f| f.key == "deleted-flag"),+ "override must be added"+ );+ assert!(+ result.cohorts.is_none(),+ "an applied override must drop the scoped cohort list so the matcher loads the full set"+ );+}++/// A REPLACED flag's historical version can depend on flags its live version+/// does not. The recompute must run for replacements too, or the cached+/// `transitive_deps` would leave the dependency unevaluated and the condition+/// would read as a silent non-match.+#[tokio::test]+async fn test_fetch_and_filter_replace_override_recomputes_metadata() {+ let redis_client = setup_redis_client(None).await;+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;+ let flag_service = FlagService::new(+ redis_client.clone(),+ reader.clone(),+ team_hypercache_reader,+ hypercache_reader,+ Arc::new(FlagDefinitionsCache::disabled()),+ NegativeCache::new(100, 300),+ false,+ );+ let context = TestContext::new(None).await;+ let team = context.insert_new_team(None).await.unwrap();++ // Two independent live flags: the cached metadata has no edges.+ let flags = vec![+ mock!(FeatureFlag,+ name: "Flag A".mock_into(),+ id: 1,+ key: "flag-a".mock_into(),+ team_id: team.id,+ filters: FlagFilters::default()+ ),+ mock!(FeatureFlag,+ name: "Flag B".mock_into(),+ id: 2,+ key: "flag-b".mock_into(),+ team_id: team.id,+ filters: FlagFilters::default()+ ),+ ];+ let eval_metadata = EvaluationMetadata {+ dependency_stages: vec![vec![1, 2]],+ flags_with_missing_deps: vec![],+ transitive_deps: HashMap::from([+ (1, std::collections::HashSet::new()),+ (2, std::collections::HashSet::new()),+ ]),+ ..Default::default()+ };+ let wrapper = HypercacheFlagsWrapper {+ flags: flags.clone(),+ evaluation_metadata: eval_metadata,+ cohorts: Some(vec![]),+ };+ let json_string = serde_json::to_string(&wrapper).unwrap();+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)+ .await+ .unwrap();++ // The historical version of flag-a depended on flag-b.+ let override_def = json!({+ "id": 1,+ "team_id": team.id,+ "name": "Flag A at the timestamp",+ "key": "flag-a",+ "filters": { "groups": [{+ "properties": [{+ "key": "2",+ "type": "flag",+ "value": "true",+ "operator": "flag_evaluates_to",+ }],+ "rollout_percentage": 100,+ }] },+ "deleted": false,+ "active": true,+ });+ let overrides: HashMap<String, Value> = HashMap::from([("flag-a".to_string(), override_def)]);++ let result = fetch_and_filter(+ &flag_service,+ team.id,+ &FlagsQueryParams::default(),+ &HeaderMap::new(),+ None,+ None,+ Some(&overrides),+ )+ .await+ .unwrap();++ assert_eq!(+ result.evaluation_metadata.transitive_deps.get(&1),+ Some(&std::collections::HashSet::from([2])),+ "replace-path override must recompute transitive deps (got {:?})",+ result.evaluation_metadata.transitive_deps+ );+ assert!(+ result.cohorts.is_none(),+ "an applied override must drop the scoped cohort list"+ );+}++/// An added flag can depend on a flag that is itself still deleted and so+/// absent from the merged list. The metadata recompute must mark the added+/// flag as missing a dependency (it then fails closed), and the live flag+/// must stay staged.+#[tokio::test]+async fn test_fetch_and_filter_override_add_with_missing_dependency_fails_closed() {+ let redis_client = setup_redis_client(None).await;+ let reader: Arc<dyn Client + Send + Sync> = setup_pg_reader_client(None);+ let team_hypercache_reader = setup_team_hypercache_reader(redis_client.clone()).await;+ let hypercache_reader = setup_hypercache_reader(redis_client.clone()).await;+ let flag_service = FlagService::new(+ redis_client.clone(),+ reader.clone(),+ team_hypercache_reader,+ hypercache_reader,+ Arc::new(FlagDefinitionsCache::disabled()),+ NegativeCache::new(100, 300),+ false,+ );+ let context = TestContext::new(None).await;+ let team = context.insert_new_team(None).await.unwrap();++ let flags = vec![mock!(FeatureFlag,+ name: "Live flag".mock_into(),+ id: 1,+ key: "live-flag".mock_into(),+ team_id: team.id,+ filters: FlagFilters::default()+ )];+ let eval_metadata = EvaluationMetadata {+ dependency_stages: vec![vec![1]],+ flags_with_missing_deps: vec![],+ transitive_deps: HashMap::from([(1, std::collections::HashSet::new())]),+ ..Default::default()+ };+ let wrapper = HypercacheFlagsWrapper {+ flags: flags.clone(),+ evaluation_metadata: eval_metadata,+ cohorts: None,+ };+ let json_string = serde_json::to_string(&wrapper).unwrap();+ write_flags_wire_json_to_redis(redis_client.clone(), team.id, json_string)+ .await+ .unwrap();++ // The added flag depends on flag id 99999, which is in no list (e.g. a+ // dependency deleted in the same bulk delete).+ let override_def = json!({+ "id": 2,+ "team_id": team.id,+ "name": "Reconstructed deleted flag",+ "key": "deleted-flag",+ "filters": { "groups": [{+ "properties": [{+ "key": "99999",+ "type": "flag",+ "value": "true",+ "operator": "flag_evaluates_to",+ }],+ "rollout_percentage": 100,+ }] },+ "deleted": false,+ "active": true,+ });+ let overrides: HashMap<String, Value> =+ HashMap::from([("deleted-flag".to_string(), override_def)]);++ let result = fetch_and_filter(+ &flag_service,+ team.id,+ &FlagsQueryParams::default(),+ &HeaderMap::new(),+ None,+ None,+ Some(&overrides),+ )+ .await+ .unwrap();++ let added = result+ .flags+ .iter()+ .find(|f| f.key == "deleted-flag")+ .expect("added flag must be in the list");+ assert!(+ result+ .evaluation_metadata+ .flags_with_missing_deps+ .contains(&added.id),+ "added flag with an unresolvable dependency must be marked missing-deps (got {:?})",+ result.evaluation_metadata.flags_with_missing_deps+ );+ let staged: Vec<i32> = result+ .evaluation_metadata+ .dependency_stages+ .iter()+ .flatten()+ .copied()+ .collect();+ assert!(+ staged.contains(&1),+ "live flag must stay staged (stages: {staged:?})"+ );+}+/// Two `fetch_and_filter` calls against a primed `FlagDefinitionsCache` must/// share the same `Arc<[FeatureFlag]>` and `Arc<EvaluationMetadata>` — i.e./// `fetch_and_filter` must not deep-clone either field.