Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): make the live flag the only gate on the stale flag check (#111767)

James Gillaaa8d9749c4f
diff --git a/posthog/api/health_issue.py b/posthog/api/health_issue.py
index aeab9a9d8a1c..9ed5e5c02d72 100644
--- a/posthog/api/health_issue.py
+++ b/posthog/api/health_issue.py
@@ -34,6 +34,8 @@ from posthog.rate_limit import HealthIssueRefreshThrottle
from posthog.utils import relative_date_parse
if TYPE_CHECKING:
+ from posthog.temporal.health_checks.framework import ObjectAccessGate
+
from products.access_control.backend.facade.user_access_control import UserAccessControl
@@ -293,12 +295,74 @@ def _kinds_hidden_by_access_control(request: Request, user_access_control: "User
}
+def _denied_object_ids_by_kind(
+ request: Request, user_access_control: "UserAccessControl"
+) -> dict[str, tuple["ObjectAccessGate", frozenset[str]]]:
+ """Check kind → (its object gate, ids of that kind's objects this user is denied).
+
+ Only kinds with at least one denied id appear. Resource-level denies are
+ `_kinds_hidden_by_access_control`'s job; this is the per-object rung under it.
+ """
+ from posthog.temporal.health_checks.framework import object_access_gates_by_kind # noqa: PLC0415
+
+ if is_service_auth(request):
+ return {}
+
+ denied_by_resource = user_access_control.blocked_resource_ids_by_scope
+ denied_by_kind: dict[str, tuple[ObjectAccessGate, frozenset[str]]] = {}
+ for kind, gate in object_access_gates_by_kind().items():
+ denied_ids = denied_by_resource.get(gate.resource)
+ if denied_ids and gate.model is not None and request.user.id is not None:
+ # `blocked_resource_ids_by_scope` has no creator exemption, while the resource's own
+ # endpoints let a creator open what they made, whatever its access level. `_base_manager`
+ # keeps soft-deleted rows that a default manager hides. The ids come from this team's
+ # access controls, so the query needs no team filter.
+ created = gate.model._base_manager.filter(pk__in=denied_ids, created_by_id=request.user.id)
+ denied_ids = denied_ids - {str(pk) for pk in created.values_list("pk", flat=True)}
+ if denied_ids:
+ denied_by_kind[kind] = (gate, denied_ids)
+ return denied_by_kind
+
+
+def _payload_id_values(denied_ids: frozenset[str]) -> list[str | int]:
+ """Both spellings of each id, because `AccessControl.resource_id` is a string and a check
+ writes the id in whatever type its model uses."""
+ values: list[str | int] = list(denied_ids)
+ values.extend(int(denied_id) for denied_id in denied_ids if denied_id.isdigit())
+ return values
+
+
+def _denied_object_filter(kind: str, gate: "ObjectAccessGate", denied_ids: frozenset[str]) -> Q:
+ """The SQL form of `_issue_names_denied_object`, for list and summary.
+
+ The `has_key` guard keeps an issue without the key listed. A missing key reads as NULL,
+ and a NULL inside the excluded condition drops the row, while retrieve reads the same
+ missing key as "not denied" and serves it.
+ """
+ return Q(
+ kind=kind,
+ payload__has_key=gate.object_key,
+ **{f"payload__{gate.object_key}__in": _payload_id_values(denied_ids)},
+ )
+
+
+def _issue_names_denied_object(
+ issue: HealthIssue, denied_by_kind: dict[str, tuple["ObjectAccessGate", frozenset[str]]]
+) -> bool:
+ entry = denied_by_kind.get(issue.kind)
+ if entry is None or not isinstance(issue.payload, dict):
+ return False
+ gate, denied_ids = entry
+ object_id = issue.payload.get(gate.object_key)
+ return object_id is not None and str(object_id) in denied_ids
+
+
class HealthIssueResourceAccessPermission(BasePermission):
- """Denies an issue whose check declared an access-controlled resource the user cannot view.
+ """Denies an issue whose check declared an access-controlled resource, or object, the user cannot view.
`health_issue` is not itself an access-controlled resource, but issue payloads
and rendered titles/summaries embed metadata about resources that are (source
- pipeline names, view names). List and summary hide such kinds in
+ pipeline names, view names, flag keys). List and summary hide such issues in
`safely_get_queryset`; this covers retrieve and the object actions with the
same 403 that `AccessControlPermission` returns on the resource's own endpoints.
"""
@@ -306,7 +370,9 @@ class HealthIssueResourceAccessPermission(BasePermission):
message = "You do not have viewer access to this resource."
def has_object_permission(self, request: Request, view, obj: HealthIssue) -> bool:
- return obj.kind not in _kinds_hidden_by_access_control(request, view.user_access_control)
+ if obj.kind in _kinds_hidden_by_access_control(request, view.user_access_control):
+ return False
+ return not _issue_names_denied_object(obj, _denied_object_ids_by_kind(request, view.user_access_control))
def _issue_counts(queryset: QuerySet) -> dict[str, Any]:
@@ -471,6 +537,8 @@ class HealthIssueViewSet(TeamAndOrgViewSetMixin, ListModelMixin, RetrieveModelMi
hidden_kinds = _kinds_hidden_by_access_control(self.request, self.user_access_control)
if hidden_kinds:
queryset = queryset.exclude(kind__in=hidden_kinds)
+ for kind, (gate, denied_ids) in _denied_object_ids_by_kind(self.request, self.user_access_control).items():
+ queryset = queryset.exclude(_denied_object_filter(kind, gate, denied_ids))
return queryset
diff --git a/posthog/api/test/test_health_issue.py b/posthog/api/test/test_health_issue.py
index 3de2e7e25991..418c4354c52d 100644
--- a/posthog/api/test/test_health_issue.py
+++ b/posthog/api/test/test_health_issue.py
@@ -15,9 +15,11 @@ from posthog.constants import AvailableFeature
from posthog.models.health_issue import HealthIssue
from posthog.models.organization import OrganizationMembership
from posthog.models.team import Team
+from posthog.models.user import User
from posthog.redis import get_client
from products.access_control.backend.models.access_control import AccessControl
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
from products.growth.backend.constants import github_sdk_versions_key
@@ -572,6 +574,73 @@ class TestHealthIssueAccessControl(APIBaseTest):
retrieve_response = self.client.get(self._url(f"/{issue.id}"))
self.assertEqual(retrieve_response.status_code, status.HTTP_200_OK)
+ def _create_stale_flag_issue(self, flag: FeatureFlag) -> HealthIssue:
+ return HealthIssue.objects.create(
+ team=self.team,
+ kind="stale_feature_flags",
+ severity=HealthIssue.Severity.INFO,
+ payload={"flag_id": flag.id, "flag_key": flag.key, "flag_name": flag.name},
+ unique_hash=f"stale-{flag.id}",
+ )
+
+ def _deny_flag(self, flag: FeatureFlag) -> None:
+ AccessControl.objects.create(
+ team=self.team, resource="feature_flag", resource_id=str(flag.id), access_level="none"
+ )
+
+ def test_member_denied_one_flag_does_not_see_its_issue_in_list_or_summary(self):
+ other_member = User.objects.create_and_join(self.organization, "[email protected]", None)
+ denied_flag = FeatureFlag.objects.create(team=self.team, key="denied-flag", created_by=other_member)
+ allowed_flag = FeatureFlag.objects.create(team=self.team, key="allowed-flag", created_by=None)
+ self._create_stale_flag_issue(denied_flag)
+ allowed_issue = self._create_stale_flag_issue(allowed_flag)
+ # Without its object key the issue names no flag, and retrieve serves it, so the list must too.
+ keyless_issue = HealthIssue.objects.create(
+ team=self.team,
+ kind="stale_feature_flags",
+ severity=HealthIssue.Severity.INFO,
+ payload={},
+ unique_hash="stale-keyless",
+ )
+ self._deny_flag(denied_flag)
+
+ list_response = self.client.get(self._url("?kind=stale_feature_flags"))
+ self.assertEqual(list_response.status_code, status.HTTP_200_OK)
+ self.assertCountEqual(
+ [result["id"] for result in list_response.json()["results"]],
+ [str(allowed_issue.id), str(keyless_issue.id)],
+ )
+
+ summary_response = self.client.get(self._url("/summary"))
+ self.assertEqual(summary_response.status_code, status.HTTP_200_OK)
+ self.assertEqual(summary_response.json()["unsnoozed"]["by_kind"], {"stale_feature_flags": 2})
+
+ def test_creator_still_sees_the_issue_for_their_own_denied_flag(self):
+ own_flag = FeatureFlag.objects.create(team=self.team, key="own-flag", created_by=self.user)
+ issue = self._create_stale_flag_issue(own_flag)
+ self._deny_flag(own_flag)
+
+ list_response = self.client.get(self._url("?kind=stale_feature_flags"))
+ self.assertEqual([result["id"] for result in list_response.json()["results"]], [str(issue.id)])
+ self.assertEqual(self.client.get(self._url(f"/{issue.id}")).status_code, status.HTTP_200_OK)
+
+ @parameterized.expand(
+ [
+ ("retrieve", lambda client, url: client.get(url)),
+ ("resolve", lambda client, url: client.post(f"{url}/resolve")),
+ ("dismiss", lambda client, url: client.patch(url, {"dismissed": True})),
+ ("snooze", lambda client, url: client.patch(url, {"snoozed_until": "P7D"})),
+ ]
+ )
+ def test_member_denied_one_flag_object_action_returns_403(self, _name, request):
+ flag = FeatureFlag.objects.create(team=self.team, key="denied-flag", created_by=None)
+ issue = self._create_stale_flag_issue(flag)
+ self._deny_flag(flag)
+
+ response = request(self.client, self._url(f"/{issue.id}"))
+ self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN)
+ self.assertEqual(response.json()["detail"], "You do not have viewer access to this resource.")
+
class TestSnoozeDurationField(SimpleTestCase):
@parameterized.expand(
diff --git a/posthog/temporal/health_checks/framework.py b/posthog/temporal/health_checks/framework.py
index f032908a345a..f64709f80478 100644
--- a/posthog/temporal/health_checks/framework.py
+++ b/posthog/temporal/health_checks/framework.py
@@ -2,7 +2,7 @@ from __future__ import annotations
import inspect
from dataclasses import dataclass
-from typing import Any
+from typing import TYPE_CHECKING, Any
from django.conf import settings
@@ -16,6 +16,9 @@ from posthog.temporal.health_checks.registry import _DETECT_FNS, HEALTH_CHECKS,
from products.signals.backend.enums import ReportPriority
+if TYPE_CHECKING:
+ from django.db.models import Model
+
# Severity → signal weight. Critical issues hit weight 1.0, which triggers the
# Signals summary pipeline; lower severities rank below in the inbox feed.
_SEVERITY_WEIGHT: dict[str, float] = {
@@ -154,6 +157,8 @@ class HealthCheckRegistration:
product: Product | None
remediation: Remediation | None
access_controlled_resource: APIScopeObject | None
+ access_controlled_object_key: str | None
+ access_controlled_object_model: type[Model] | None
def __post_init__(self) -> None:
# A fraction, not a percent: 50 here would silently skip the rollout filter and hit every team.
@@ -166,6 +171,18 @@ class HealthCheckRegistration:
f"HealthCheckRegistration not_processed_threshold must be between 0 and 1, "
f"got {self.not_processed_threshold}"
)
+ # The denied ids are looked up under the resource, so a key alone would gate nobody.
+ if self.access_controlled_object_key is not None and self.access_controlled_resource is None:
+ raise ValueError(
+ f"HealthCheckRegistration access_controlled_object_key={self.access_controlled_object_key!r} "
+ "requires access_controlled_resource"
+ )
+ # The creator exemption only lifts an object-level deny, so it means nothing without one.
+ if self.access_controlled_object_model is not None and self.access_controlled_object_key is None:
+ raise ValueError(
+ f"HealthCheckRegistration access_controlled_object_model={self.access_controlled_object_model!r} "
+ "requires access_controlled_object_key"
+ )
def _register_health_check(cls: type[HealthCheck]) -> None:
@@ -187,6 +204,8 @@ def _register_health_check(cls: type[HealthCheck]) -> None:
product=cls.product,
remediation=cls.remediation,
access_controlled_resource=cls.access_controlled_resource,
+ access_controlled_object_key=cls.access_controlled_object_key,
+ access_controlled_object_model=cls.access_controlled_object_model,
)
HEALTH_CHECKS[cls.kind] = registration
@@ -204,6 +223,13 @@ class HealthCheck:
not_processed_threshold: float = 0.1
dry_run: bool = False
active_since_days: int | None = DEFAULT_ACTIVE_SINCE_DAYS
+ # False stops the `$health_check_issue_firing` and `$health_check_issue_resolved` events for
+ # this kind. Issues are still written, served by the Health API, and sent to the Signals inbox
+ # when the check overrides `render_signal`. Use it for a check that writes many issues per
+ # team, because the stock health-alert destination does not filter on kind and sends one
+ # notification per issue. The alert emitter also ignores the access-control fields below, so
+ # this is the only way to keep a gated payload out of destinations.
+ emits_alerts: bool = True
# Static, kind-level guidance on how to resolve issues of this kind. Unlike
# `render_alert` (which describes a *specific* issue), this is the same for
@@ -227,13 +253,31 @@ class HealthCheck:
# Health API hides the check's issues from members whose access to the
# resource is restricted. None means every team member sees the issues.
#
- # Two limits. The gate is resource-level only: an `AccessControl` deny scoped
- # to one object's `resource_id` is not consulted, so a member barred from a
- # single source or view still sees its issues. And only the Health API reads
- # this field: the Signals inbox emitter does not, so a check that also
- # overrides `render_signal` still publishes its payload there.
+ # Two limits. On its own the gate is resource-level only: an `AccessControl`
+ # deny scoped to one object's `resource_id` is not consulted, so a member
+ # barred from a single source or view still sees its issues. Set
+ # `access_controlled_object_key` as well to close that. And only the Health
+ # API reads these fields. The alert emitter does not, so set `emits_alerts`
+ # to False to keep the payload out of destinations. The Signals inbox emitter
+ # does not either, so a check that also overrides `render_signal` still
+ # publishes its payload there.
access_controlled_resource: APIScopeObject | None = None
+ # The payload key that holds the id of the one object an issue describes, for
+ # example "flag_id" under the "feature_flag" resource. When set, the Health API
+ # also hides an issue from a member whose access to that object resolves to
+ # "none", even though they can view the resource in general. The payload value
+ # and the `AccessControl.resource_id` are compared as strings, so an integer id
+ # in the payload matches. Requires `access_controlled_resource`.
+ access_controlled_object_key: str | None = None
+
+ # The model the object ids belong to. When set, the Health API reads its
+ # `created_by_id` and still shows a creator the issues about objects they are
+ # denied, to match the creator exemption on the resource's own endpoints,
+ # where a creator can always open what they made. Requires
+ # `access_controlled_object_key`.
+ access_controlled_object_model: type[Model] | None = None
+
def __init_subclass__(cls, **kwargs: object) -> None:
super().__init_subclass__(**kwargs)
if not hasattr(cls, "name") or not hasattr(cls, "kind"):
@@ -327,6 +371,31 @@ def access_controlled_resources_by_kind() -> dict[str, APIScopeObject]:
}
+@dataclass(frozen=True)
+class ObjectAccessGate:
+ resource: APIScopeObject
+ object_key: str
+ model: type[Model] | None
+
+
+def object_access_gates_by_kind() -> dict[str, ObjectAccessGate]:
+ """Map check kind → how its issues are gated on the one object each describes.
+
+ Only kinds that declare both `access_controlled_resource` and
+ `access_controlled_object_key` appear.
+ """
+ ensure_registry_loaded()
+ return {
+ kind: ObjectAccessGate(
+ resource=registration.access_controlled_resource,
+ object_key=registration.access_controlled_object_key,
+ model=registration.access_controlled_object_model,
+ )
+ for kind, registration in HEALTH_CHECKS.items()
+ if registration.access_controlled_resource is not None and registration.access_controlled_object_key is not None
+ }
+
+
def remediation_for_kind(kind: str) -> Remediation | None:
"""Return the static, kind-level remediation guide, or None if there isn't one.
diff --git a/posthog/temporal/health_checks/processing.py b/posthog/temporal/health_checks/processing.py
index 519fbae26cc2..52ac54329175 100644
--- a/posthog/temporal/health_checks/processing.py
+++ b/posthog/temporal/health_checks/processing.py
@@ -78,10 +78,11 @@ def _process_batch_detection(
result.issues_resolved = len(newly_resolved)
result.resolve_duration = time.monotonic() - start
- for issue in newly_active:
- emit_health_check_alert(issue, status="firing")
+ if check_cls is None or check_cls.emits_alerts:
+ for issue in newly_active:
+ emit_health_check_alert(issue, status="firing")
+ for issue in newly_resolved:
+ emit_health_check_alert(issue, status="resolved")
emit_health_check_signals(newly_active)
- for issue in newly_resolved:
- emit_health_check_alert(issue, status="resolved")
return result
diff --git a/posthog/temporal/health_checks/tests/test_framework.py b/posthog/temporal/health_checks/tests/test_framework.py
index eb268eba9dbe..af3d243ad932 100644
--- a/posthog/temporal/health_checks/tests/test_framework.py
+++ b/posthog/temporal/health_checks/tests/test_framework.py
@@ -31,6 +31,8 @@ def _registration(**overrides: Any) -> HealthCheckRegistration:
"product": None,
"remediation": None,
"access_controlled_resource": None,
+ "access_controlled_object_key": None,
+ "access_controlled_object_model": None,
}
fields.update(overrides)
return HealthCheckRegistration(**fields)
@@ -64,6 +66,21 @@ def test_rejects_out_of_range_fractions(field: str, value: float) -> None:
_registration(**{field: value})
+ "overrides,missing",
+ [
+ ({"access_controlled_object_key": "flag_id"}, "access_controlled_resource"),
+ (
+ {"access_controlled_resource": "feature_flag", "access_controlled_object_model": HealthIssue},
+ "access_controlled_object_key",
+ ),
+ ],
+)
+def test_an_access_key_without_the_field_it_narrows_is_rejected(overrides: dict[str, Any], missing: str) -> None:
+ with pytest.raises(ValueError, match=f"requires {missing}"):
+ _registration(**overrides)
+
+
def test_declared_access_controlled_resources_have_access_controls() -> None:
# A resource outside these sets resolves to the "editor" default, which
# satisfies "viewer" and silently leaves the check's issues visible to
diff --git a/posthog/temporal/health_checks/tests/test_signal_emitter.py b/posthog/temporal/health_checks/tests/test_signal_emitter.py
index 8e86634341d8..91cfaf3ceea0 100644
--- a/posthog/temporal/health_checks/tests/test_signal_emitter.py
+++ b/posthog/temporal/health_checks/tests/test_signal_emitter.py
@@ -1,4 +1,4 @@
-from unittest.mock import AsyncMock, MagicMock, patch
+from unittest.mock import AsyncMock, MagicMock, call, patch
from django.test import SimpleTestCase
@@ -194,9 +194,10 @@ class TestSeamEmitsSignalsOnFiringOnly(SimpleTestCase):
patch(
"posthog.temporal.health_checks.processing.resolve_stale_issues_with_deltas", return_value=[resolved]
),
- patch("posthog.temporal.health_checks.processing.emit_health_check_alert"),
+ patch("posthog.temporal.health_checks.processing.emit_health_check_alert") as emit_alert,
patch("posthog.temporal.health_checks.processing.emit_health_check_signals") as emit_signals,
):
_process_batch_detection([42], "k", lambda team_ids: {}, dry_run=dry_run)
emit_signals.assert_called_once_with([firing])
+ emit_alert.assert_has_calls([call(firing, status="firing"), call(resolved, status="resolved")])
diff --git a/products/feature_flags/backend/temporal/health_checks/stale_flags.py b/products/feature_flags/backend/temporal/health_checks/stale_flags.py
index cbc4757cdae3..de75c680ac27 100644
--- a/products/feature_flags/backend/temporal/health_checks/stale_flags.py
+++ b/products/feature_flags/backend/temporal/health_checks/stale_flags.py
@@ -66,18 +66,16 @@ class StaleFeatureFlagsCheck(HealthCheck):
owner = JobOwners.TEAM_FEATURE_FLAGS
product = Product.FEATURE_FLAGS
schedule = "0 6 * * 1" # weekly, Mondays 06:00 UTC
- # Payloads carry flag keys and names.
+ # Each payload carries one flag's key, name and rollout state.
access_controlled_resource = "feature_flag"
+ access_controlled_object_key = "flag_id"
+ access_controlled_object_model = FeatureFlag
+ # A team's first run writes one issue per stale flag, and alerts would bypass the per-flag gate
+ # above. The feature-flags scout reads these issues through the Health API instead.
+ emits_alerts = False
# Postgres-heavy and one issue per stale flag rather than per team, so smaller
# batches than the default policy.
policy = HealthExecutionPolicy(batch_size=250, max_concurrent=2)
- # Both stay for one more deploy, so the gate reaches every worker before anything can write.
- # Web's migrate job copies these into the schedule's workflow inputs, and the health-check
- # worker deploys as a separate app behind it. A worker still on the previous release has no
- # `eligible_team_ids`, so removing them in this release would let a run that starts inside
- # that window write live issues for every active team. The follow-up removes both.
- dry_run = True
- rollout_percentage = 0.01
remediation = Remediation(
human="""
Open the flag and confirm the staleness evidence is still current. Check every
diff --git a/products/feature_flags/backend/temporal/health_checks/tests/test_stale_flags.py b/products/feature_flags/backend/temporal/health_checks/tests/test_stale_flags.py
index ef346ac544db..7a3292e5dc80 100644
--- a/products/feature_flags/backend/temporal/health_checks/tests/test_stale_flags.py
+++ b/products/feature_flags/backend/temporal/health_checks/tests/test_stale_flags.py
@@ -883,7 +883,7 @@ class TestStaleFlagsDetect(BaseTest):
assert len(after) == len(before)
@patch("posthog.temporal.health_checks.processing.emit_health_check_alert")
- def test_issue_lifecycle(self, _mock_alert) -> None:
+ def test_issue_lifecycle(self, mock_alert) -> None:
flag_a = self._create_flag("lifecycle-a", **stale_by_usage())
flag_b = self._create_flag("lifecycle-b", **stale_by_usage())
check = StaleFeatureFlagsCheck()
@@ -931,6 +931,9 @@ class TestStaleFlagsDetect(BaseTest):
run(gate_answer=False)
assert active_issues().count() == 2
+ # The runs above fired and resolved issues, and none of them reached a destination.
+ mock_alert.assert_not_called()
+
class TestStaleFlagsContract(SimpleTestCase):
def _issue(self, payload: dict[str, Any]) -> HealthIssue:
@@ -942,20 +945,23 @@ class TestStaleFlagsContract(SimpleTestCase):
unique_hash="h",
)
- def test_registered_dry_until_the_gate_reaches_every_worker(self) -> None:
+ def test_registered_with_the_live_flag_as_the_only_gate(self) -> None:
ensure_registry_loaded()
registration = HEALTH_CHECKS["stale_feature_flags"]
assert registration.owner == JobOwners.TEAM_FEATURE_FLAGS
assert registration.product == Product.FEATURE_FLAGS
- # Web writes these into the schedule before the worker redeploys, so a worker without
- # `eligible_team_ids` must still find a dry registration. The follow-up drops both and
- # leaves the flag as the only gate.
- assert registration.dry_run is True
- assert registration.rollout_percentage == 0.01
+ # These are the framework defaults. A `rollout_percentage` below 1.0 samples teams out before
+ # `eligible_team_ids` runs, and `dry_run` drops the writes after detection. Overriding either
+ # one means a team the flag enables can still get no issues.
+ assert registration.dry_run is False
+ assert registration.rollout_percentage == 1.0
assert registration.schedule == "0 6 * * 1"
assert registration.remediation is not None
- # Payloads carry flag keys and names, so the Health API must gate them on flag access.
+ # Payloads carry one flag's key, name and rollout state, so the Health API must gate each
+ # issue on access to that flag, not only on access to flags as a resource.
assert registration.access_controlled_resource == "feature_flag"
+ assert registration.access_controlled_object_key == "flag_id"
+ assert registration.access_controlled_object_model is FeatureFlag
def test_remediation_orders_code_removal_before_archive(self) -> None:
remediation = StaleFeatureFlagsCheck.remediation