Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): notify teams whose flags target deleted cohorts (#118840)

Phil Haackc123fd249975
diff --git a/frontend/src/lib/components/NotificationsMenu/NotificationRow.tsx b/frontend/src/lib/components/NotificationsMenu/NotificationRow.tsx
index a3e509d42f9f..776146c4cbdd 100644
--- a/frontend/src/lib/components/NotificationsMenu/NotificationRow.tsx
+++ b/frontend/src/lib/components/NotificationsMenu/NotificationRow.tsx
@@ -81,6 +81,10 @@ export const REALTIME_NOTIFICATION_TYPE_META: Record<string, { label: string; de
label: 'Workflow suggestions',
description: 'When PostHog suggests a change to a workflow you set up',
},
+ feature_flag_failure: {
+ label: 'Feature flag failures',
+ description: 'When a feature flag in your project fails to evaluate',
+ },
}
export function NotificationTitle({
diff --git a/frontend/src/lib/components/NotificationsMenu/notificationToasts.tsx b/frontend/src/lib/components/NotificationsMenu/notificationToasts.tsx
index 8a3d8efacda3..21cdea092681 100644
--- a/frontend/src/lib/components/NotificationsMenu/notificationToasts.tsx
+++ b/frontend/src/lib/components/NotificationsMenu/notificationToasts.tsx
@@ -14,6 +14,7 @@ import {
IconPlug,
IconSparkles,
IconStar,
+ IconToggle,
IconTrending,
IconWarning,
} from '@posthog/icons'
@@ -40,6 +41,7 @@ const NOTIFICATION_TYPE_ICONS: Record<string, { Icon: ComponentType<{ className?
subscription_nudge: { Icon: IconBell, color: 'text-primary' },
data_quality_check_failure: { Icon: IconWarning, color: 'text-danger' },
workflow_suggestion: { Icon: IconSparkles, color: 'text-primary' },
+ feature_flag_failure: { Icon: IconToggle, color: 'text-danger' },
}
export function getNotificationIcon(notificationType: string, className: string = 'size-5'): JSX.Element {
diff --git a/products/feature_flags/backend/deleted_cohort_notices.py b/products/feature_flags/backend/deleted_cohort_notices.py
new file mode 100644
index 000000000000..52446be84112
--- /dev/null
+++ b/products/feature_flags/backend/deleted_cohort_notices.py
@@ -0,0 +1,385 @@
+"""Find active flags whose release conditions target a deleted cohort, and build one in-app notice per team.
+
+The flag evaluator loads only the non-deleted cohorts of the flag's project. A release condition
+on any other cohort id fails the flag for every user that the condition is evaluated for. The
+flag API rejects these conditions on new writes. This module finds the flags that already hold one.
+
+It checks only the cohorts a flag names directly. A live cohort that includes a deleted cohort also
+fails the flag, but cohort saves reject that reference and the cohort delete guard blocks deleting
+an included cohort, so no new case can arise.
+"""
+
+from collections import defaultdict
+from collections.abc import Collection, Iterator, Mapping, Sequence
+from dataclasses import replace
+from datetime import UTC, datetime
+from typing import Any, Literal
+
+from django.db.models import Model, QuerySet
+
+import posthoganalytics
+
+from posthog.dataclasses import frozen
+from posthog.models import Team, User
+from posthog.scopes import APIScopeObject
+from posthog.utils import human_list
+
+from products.access_control.backend.facade.user_access_control import UserAccessControl
+from products.cohorts.backend.models.cohort import Cohort
+from products.feature_flags.backend.facade.config import ConfigFormatError, decode_config
+from products.feature_flags.backend.facade.references import references
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+from products.notifications.backend.facade.api import (
+ NotificationData,
+ NotificationType,
+ RecipientsResolver,
+ SourceType,
+ TargetType,
+)
+from products.product_tours.backend.models import ProductTour
+from products.surveys.backend.models import Survey
+
+# A fixed key gives each team at most one notice. create_notification returns the existing event on a re-run.
+IDEMPOTENCY_KEY = "flags-deleted-cohort-notice"
+TITLE_MAX_LENGTH = 255
+MAX_LISTED_FLAGS = 5
+PAGE_SIZE = 500
+
+OwnerKind = Literal["survey", "product_tour"]
+
+
+@frozen
+class DeletedCohort:
+ id: int
+ name: str | None
+ # True when the id names no cohort in the flag's project. False when the cohort is soft-deleted.
+ missing: bool
+
+
+@frozen
+class FlagOwner:
+ """The survey or product tour that manages a flag. The flag list hides these flags."""
+
+ kind: OwnerKind
+ id: str
+ name: str
+
+
+@frozen
+class AffectedFlag:
+ flag_id: int
+ team_id: int
+ key: str
+ last_called_at: datetime | None
+ cohorts: tuple[DeletedCohort, ...]
+ owner: FlagOwner | None
+
+
+@frozen
+class DeliveryPreview:
+ notifications_enabled: bool
+ # An upper bound, because create_notification also drops recipients who muted this notification type.
+ recipient_count: int
+
+
+@frozen
+class _FlagRow:
+ id: int
+ team_id: int
+ project_id: int
+ key: str
+ filters: Any
+ last_called_at: datetime | None
+
+
+@frozen
+class _StoredCohort:
+ project_id: int
+ deleted: bool
+ name: str | None
+
+
+def find_flags_on_deleted_cohorts(team_ids: Collection[int] | None = None) -> dict[int, list[AffectedFlag]]:
+ """Active flags whose release conditions target a deleted or missing cohort, keyed by team id."""
+ # The default manager excludes deleted flags. There is no SQL pre-filter on `filters`, because every
+ # read of the stored document goes through the config facade, which checks its format first.
+ flags = FeatureFlag.objects.filter(active=True)
+ if team_ids is not None:
+ flags = flags.filter(team_id__in=team_ids)
+ affected: dict[int, list[AffectedFlag]] = defaultdict(list)
+ for page in _flag_pages(flags):
+ for flag in _affected_flags(page):
+ affected[flag.team_id].append(flag)
+ return dict(affected)
+
+
+def _flag_pages(flags: QuerySet[FeatureFlag]) -> Iterator[list[_FlagRow]]:
+ # The first query returns ids only. Each page then loads its rows by primary key. Production
+ # connects through PgBouncer with server-side cursors disabled, so a single query over `filters`
+ # would hold every row in memory.
+ flag_ids = list(flags.order_by("id").values_list("id", flat=True))
+ for start in range(0, len(flag_ids), PAGE_SIZE):
+ rows = (
+ # nosemgrep: idor-lookup-without-team -- operator-run scan across every team. The ids come from the query above.
+ FeatureFlag.objects.filter(pk__in=flag_ids[start : start + PAGE_SIZE])
+ .order_by("id")
+ .values_list("id", "team_id", "team__project_id", "key", "filters", "last_called_at")
+ )
+ yield [
+ _FlagRow(
+ id=flag_id,
+ team_id=team_id,
+ project_id=project_id,
+ key=key,
+ filters=filters,
+ last_called_at=last_called_at,
+ )
+ for flag_id, team_id, project_id, key, filters, last_called_at in rows
+ ]
+
+
+def _affected_flags(page: Sequence[_FlagRow]) -> Iterator[AffectedFlag]:
+ cohort_ids_by_flag = {row.id: _direct_cohort_ids(row.filters) for row in page}
+ stored_cohorts = {
+ cohort_id: _StoredCohort(project_id=project_id, deleted=deleted, name=name)
+ # nosemgrep: idor-lookup-without-team -- operator-run scan across every team. _deleted_cohort compares each cohort's project with its flag's project.
+ for cohort_id, project_id, deleted, name in Cohort.objects.filter(
+ pk__in={cohort_id for ids in cohort_ids_by_flag.values() for cohort_id in ids}
+ ).values_list("id", "team__project_id", "deleted", "name")
+ }
+ broken: list[tuple[_FlagRow, tuple[DeletedCohort, ...]]] = []
+ for row in page:
+ deleted = tuple(
+ cohort
+ for cohort_id in cohort_ids_by_flag[row.id]
+ if (cohort := _deleted_cohort(cohort_id, row.project_id, stored_cohorts)) is not None
+ )
+ if deleted:
+ broken.append((row, deleted))
+ owners = _owners([row.id for row, _ in broken])
+ for row, deleted in broken:
+ yield AffectedFlag(
+ flag_id=row.id,
+ team_id=row.team_id,
+ key=row.key,
+ last_called_at=row.last_called_at,
+ cohorts=deleted,
+ owner=owners.get(row.id),
+ )
+
+
+def _direct_cohort_ids(filters: Any) -> tuple[int, ...]:
+ try:
+ return references(decode_config(filters)).cohort_ids
+ except (ConfigFormatError, AttributeError, TypeError):
+ # The evaluator cannot read a malformed document either. That failure is out of scope here.
+ return ()
+
+
+def _deleted_cohort(cohort_id: int, project_id: int, stored: Mapping[int, _StoredCohort]) -> DeletedCohort | None:
+ cohort = stored.get(cohort_id)
+ if cohort is None or cohort.project_id != project_id:
+ return DeletedCohort(id=cohort_id, name=None, missing=True)
+ if cohort.deleted:
+ return DeletedCohort(id=cohort_id, name=cohort.name, missing=False)
+ return None
+
+
+def _owners(flag_ids: Collection[int]) -> dict[int, FlagOwner]:
+ # The same owning relations as `_OWNING_ACCESSORS` in ownership.py, limited to the owners whose
+ # flags the flag list hides. A batch query needs the forward field names, not the reverse accessors.
+ owners: dict[int, FlagOwner] = {}
+ if not flag_ids:
+ return owners
+ for field in ("targeting_flag_id", "internal_targeting_flag_id", "internal_response_sampling_flag_id"):
+ for flag_id, survey_id, name in Survey.objects.filter(**{f"{field}__in": flag_ids}).values_list(
+ field, "id", "name"
+ ):
+ owners[flag_id] = FlagOwner(kind="survey", id=str(survey_id), name=name)
+ for flag_id, tour_id, name in ProductTour.all_objects.filter(internal_targeting_flag_id__in=flag_ids).values_list(
+ "internal_targeting_flag_id", "id", "name"
+ ):
+ owners[flag_id] = FlagOwner(kind="product_tour", id=str(tour_id), name=name)
+ return owners
+
+
+class _ViewersOfListedObjects(RecipientsResolver):
+ """Narrow team recipients to the members who can view every object the notice names.
+
+ create_notification checks access to the notice's one resource type and ignores per-object
+ rules. A notice can name surveys and flags together. A member can be denied one flag.
+ This resolver narrows the recipients first. create_notification then applies its own check.
+ """
+
+ def __init__(self, flags: Sequence[AffectedFlag]) -> None:
+ self._flags = tuple(flags)
+
+ def resolve(self, target_type: TargetType, target_id: str, team_id: int | None) -> list[int]:
+ user_ids = super().resolve(target_type, target_id, team_id)
+ team = Team.objects.filter(id=team_id).first() if team_id is not None else None
+ if not user_ids or team is None:
+ return []
+ # The body names the survey for a survey's flag, and the flag for a plain or product tour flag.
+ survey_ids = [flag.owner.id for flag in self._flags if flag.owner and flag.owner.kind == "survey"]
+ flag_ids = [flag.flag_id for flag in self._flags if not (flag.owner and flag.owner.kind == "survey")]
+ named: list[Model] = [
+ *Survey.objects.filter(team__project_id=team.project_id, id__in=survey_ids),
+ *FeatureFlag.objects.filter(team__project_id=team.project_id, id__in=flag_ids),
+ ]
+ if survey_ids:
+ user_ids = self.filter_by_access_control(user_ids, "survey", team)
+ if flag_ids:
+ user_ids = self.filter_by_access_control(user_ids, "feature_flag", team)
+ viewers = []
+ for user in User.objects.filter(id__in=user_ids):
+ access = UserAccessControl(user, team)
+ if access.is_organization_admin or all(
+ access.check_access_level_for_object(obj, required_level="viewer") for obj in named
+ ):
+ viewers.append(user.id)
+ return viewers
+
+
+def build_notice(team_id: int, flags: Sequence[AffectedFlag]) -> NotificationData:
+ notice = _single_flag_notice(team_id, flags[0]) if len(flags) == 1 else _several_flags_notice(team_id, flags)
+ return replace(notice, resolver=_ViewersOfListedObjects(flags))
+
+
+def _single_flag_notice(team_id: int, flag: AffectedFlag) -> NotificationData:
+ cohorts = _cohorts_phrase(flag.cohorts)
+ those = "those cohorts" if len(flag.cohorts) > 1 else "that cohort"
+ match flag.owner:
+ case FlagOwner(kind="survey", id=survey_id, name=name):
+ return _notice(
+ team_id,
+ title=f"Survey targets a deleted cohort: {name}",
+ body=(
+ f"Survey '{name}' targets {cohorts}. Its targeting fails to evaluate until you edit the "
+ f"survey's targeting to remove {those}."
+ ),
+ resource_type="survey",
+ source_type=SourceType.SURVEY,
+ source_id=survey_id,
+ )
+ case FlagOwner(kind="product_tour", id=tour_id, name=name):
+ return _notice(
+ team_id,
+ title=f"Product tour targets a deleted cohort: {name}",
+ body=(
+ f"Product tour '{name}' targets {cohorts}. Its targeting fails to evaluate until you edit the "
+ f"tour's targeting to remove {those}."
+ ),
+ # Product tours are not an access-control resource, so the notice filters recipients on flag access.
+ resource_type="feature_flag",
+ source_url=f"/product_tours/{tour_id}",
+ )
+ case _:
+ fix = (
+ "remove those conditions or choose other cohorts"
+ if len(flag.cohorts) > 1
+ else "remove that condition or choose another cohort"
+ )
+ return _notice(
+ team_id,
+ title=f"Feature flag targets a deleted cohort: {flag.key}",
+ body=f"Flag '{flag.key}' targets {cohorts}. The flag fails to evaluate until you {fix}.",
+ resource_type="feature_flag",
+ resource_id=str(flag.flag_id),
+ source_type=SourceType.FEATURE_FLAG,
+ source_id=str(flag.flag_id),
+ )
+
+
+def _several_flags_notice(team_id: int, flags: Sequence[AffectedFlag]) -> NotificationData:
+ kinds = {flag.owner.kind if flag.owner else None for flag in flags}
+ resource_type: APIScopeObject = "feature_flag"
+ noun, source_url = "feature flags", "/feature_flags"
+ if kinds == {"survey"}:
+ noun, source_url, resource_type = "surveys", "/surveys", "survey"
+ elif kinds == {"product_tour"}:
+ noun, source_url = "product tours", "/product_tours"
+ # The most recently evaluated flags matter most to the team, so they come first in the capped list.
+ listed = sorted(flags, key=lambda flag: flag.last_called_at or datetime.min.replace(tzinfo=UTC), reverse=True)
+ items = [_item_label(flag) for flag in listed[:MAX_LISTED_FLAGS]]
+ if len(flags) > MAX_LISTED_FLAGS:
+ items.append(f"and {len(flags) - MAX_LISTED_FLAGS} more")
+ body = (
+ "Each of these targets a deleted cohort and fails to evaluate until you remove that cohort or choose "
+ f"another: {', '.join(items)}."
+ )
+ owner_nouns = [kind.replace("_", " ") for kind in ("survey", "product_tour") if kind in kinds]
+ if owner_nouns:
+ body += f" Change a {' or '.join(owner_nouns)} in its own targeting settings."
+ return _notice(
+ team_id,
+ title=f"{len(flags)} {noun} target deleted cohorts",
+ body=body,
+ resource_type=resource_type,
+ source_url=source_url,
+ )
+
+
+def _notice(
+ team_id: int,
+ *,
+ title: str,
+ body: str,
+ resource_type: APIScopeObject,
+ resource_id: str = "",
+ source_type: SourceType | None = None,
+ source_id: str | None = None,
+ source_url: str = "",
+) -> NotificationData:
+ return NotificationData(
+ team_id=team_id,
+ notification_type=NotificationType.FEATURE_FLAG_FAILURE,
+ title=title[:TITLE_MAX_LENGTH],
+ body=body,
+ target_type=TargetType.TEAM,
+ target_id=str(team_id),
+ resource_type=resource_type,
+ resource_id=resource_id,
+ source_type=source_type,
+ source_id=source_id,
+ source_url=source_url,
+ idempotency_key=IDEMPOTENCY_KEY,
+ )
+
+
+def _item_label(flag: AffectedFlag) -> str:
+ noun, name = (flag.owner.kind.replace("_", " "), flag.owner.name) if flag.owner else ("flag", flag.key)
+ return f"{noun} '{name}' ({_cohort_list(flag.cohorts)})"
+
+
+def _cohorts_phrase(cohorts: Sequence[DeletedCohort]) -> str:
+ if len(cohorts) == 1 and cohorts[0].missing:
+ return f"a cohort that no longer exists (ID {cohorts[0].id})"
+ return f"the deleted {_cohort_list(cohorts)}"
+
+
+def _cohort_list(cohorts: Sequence[DeletedCohort]) -> str:
+ noun = "cohorts" if len(cohorts) > 1 else "cohort"
+ return f"{noun} {human_list([_cohort_label(cohort) for cohort in cohorts])}"
+
+
+def _cohort_label(cohort: DeletedCohort) -> str:
+ return f"'{cohort.name}'" if cohort.name else f"ID {cohort.id}"
+
+
+def preview_delivery(team: Team, notice: NotificationData) -> DeliveryPreview:
+ """Whether create_notification would deliver this notice, and to how many people at most."""
+ organization_id = str(team.organization_id)
+ # create_notification runs the same check and returns None when the flag is off.
+ if not posthoganalytics.feature_enabled(
+ "real-time-notifications",
+ organization_id,
+ groups={"organization": organization_id},
+ only_evaluate_locally=False,
+ send_feature_flag_events=False,
+ ):
+ return DeliveryPreview(notifications_enabled=False, recipient_count=0)
+ resolver = notice.resolver or RecipientsResolver()
+ user_ids = resolver.resolve(notice.target_type, notice.target_id, team.id)
+ if notice.resource_type:
+ user_ids = resolver.filter_by_access_control(user_ids, str(notice.resource_type), team)
+ return DeliveryPreview(notifications_enabled=True, recipient_count=len(user_ids))
diff --git a/products/feature_flags/backend/management/commands/notify_flags_on_deleted_cohorts.py b/products/feature_flags/backend/management/commands/notify_flags_on_deleted_cohorts.py
new file mode 100644
index 000000000000..b9b1464fc2d1
--- /dev/null
+++ b/products/feature_flags/backend/management/commands/notify_flags_on_deleted_cohorts.py
@@ -0,0 +1,107 @@
+"""Tell each team whose active flags target a deleted cohort, with one in-app notice per team.
+
+The dry run is the default. It prints each team's flags, the notice it would get, and whether the
+notice would reach anyone. `--send` needs `--team-id`, so a send covers only the teams someone
+reviewed in a dry run.
+"""
+
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandError, CommandParser
+from django.utils import timezone
+
+from posthog.models import Team
+from posthog.utils import pluralize
+
+from products.feature_flags.backend.deleted_cohort_notices import (
+ AffectedFlag,
+ DeletedCohort,
+ build_notice,
+ find_flags_on_deleted_cohorts,
+ preview_delivery,
+)
+from products.notifications.backend.facade.api import create_notification
+
+
+class Command(BaseCommand):
+ help = "Notify teams whose active feature flags target a deleted cohort (dry run by default)"
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, nargs="+", help="Only scan these teams")
+ parser.add_argument("--send", action="store_true", help="Create the notices (requires --team-id)")
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ team_ids: list[int] | None = options["team_id"]
+ send: bool = options["send"]
+ if send and not team_ids:
+ raise CommandError("--send requires --team-id with the teams approved from a dry run")
+
+ affected = find_flags_on_deleted_cohorts(team_ids)
+ for team_id in team_ids or []:
+ if team_id not in affected:
+ self.stdout.write(f"Team {team_id}: no active flag targets a deleted cohort. Skipped.")
+
+ deliverable: list[int] = []
+ sent = 0
+ started_at = timezone.now()
+ teams = Team.objects.in_bulk(affected.keys())
+ for team_id, flags in sorted(affected.items()):
+ team = teams[team_id]
+ notice = build_notice(team_id, flags)
+ self.stdout.write(
+ f"\nTeam {team_id} ({team.name}), organization {team.organization_id}: {pluralize(len(flags), 'flag')}"
+ )
+ for flag in flags:
+ self.stdout.write(f" {_describe_flag(flag)}")
+ self.stdout.write(f" Title: {notice.title}")
+ self.stdout.write(f" Body: {notice.body}")
+ if notice.source_type:
+ self.stdout.write(f" Link: {notice.source_type.value} {notice.source_id}")
+ else:
+ self.stdout.write(f" Link: {notice.source_url}")
+ if send:
+ event = create_notification(notice)
+ if event is None:
+ self.stdout.write(
+ " Not delivered: real-time notifications are off for this organization, "
+ "or nobody in the project can view it."
+ )
+ elif event.created_at < started_at:
+ # The idempotency key returns the notice an earlier run created, unchanged.
+ self.stdout.write(f" Already sent notice {event.id} on {event.created_at.date().isoformat()}.")
+ else:
+ sent += 1
+ self.stdout.write(
+ f" Sent notice {event.id} to {pluralize(len(event.resolved_user_ids), 'person', 'people')}."
+ )
+ continue
+ preview = preview_delivery(team, notice)
+ if not preview.notifications_enabled:
+ self.stdout.write(" Would not deliver: real-time notifications are off for this organization.")
+ elif not preview.recipient_count:
+ self.stdout.write(" Would not deliver: nobody in the project can view it.")
+ else:
+ deliverable.append(team_id)
+ self.stdout.write(f" Would deliver to up to {pluralize(preview.recipient_count, 'person', 'people')}.")
+
+ flag_count = sum(len(flags) for flags in affected.values())
+ self.stdout.write(f"\n{pluralize(len(affected), 'team')}, {pluralize(flag_count, 'flag')}.")
+ if send:
+ self.stdout.write(f"Sent {pluralize(sent, 'notice')}.")
+ else:
+ self.stdout.write(f"{pluralize(len(deliverable), 'team')} would receive the notice.")
+ if deliverable:
+ self.stdout.write(f"--team-id {' '.join(str(team_id) for team_id in deliverable)}")
+
+
+def _describe_flag(flag: AffectedFlag) -> str:
+ owner = f" ({flag.owner.kind.replace('_', ' ')} '{flag.owner.name}' {flag.owner.id})" if flag.owner else ""
+ called = flag.last_called_at.date().isoformat() if flag.last_called_at else "never"
+ cohorts = ", ".join(_describe_cohort(cohort) for cohort in flag.cohorts)
+ return f"flag {flag.flag_id} '{flag.key}'{owner}, last called {called}: {cohorts}"
+
+
+def _describe_cohort(cohort: DeletedCohort) -> str:
+ if cohort.missing:
+ return f"missing cohort {cohort.id}"
+ return f"deleted cohort {cohort.id} '{cohort.name or ''}'"
diff --git a/products/feature_flags/backend/management/commands/test/test_notify_flags_on_deleted_cohorts.py b/products/feature_flags/backend/management/commands/test/test_notify_flags_on_deleted_cohorts.py
new file mode 100644
index 000000000000..20ad0da78648
--- /dev/null
+++ b/products/feature_flags/backend/management/commands/test/test_notify_flags_on_deleted_cohorts.py
@@ -0,0 +1,316 @@
+from io import StringIO
+from operator import attrgetter
+from typing import Any
+
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from django.core.management import call_command
+from django.core.management.base import CommandError
+from django.test import SimpleTestCase
+
+from parameterized import parameterized
+
+from posthog.constants import AvailableFeature
+from posthog.models import OrganizationMembership, Team, User
+
+from products.access_control.backend.models.access_control import AccessControl
+from products.cohorts.backend.models.cohort import Cohort
+from products.feature_flags.backend.deleted_cohort_notices import (
+ AffectedFlag,
+ DeletedCohort,
+ FlagOwner,
+ build_notice,
+ find_flags_on_deleted_cohorts,
+)
+from products.feature_flags.backend.models.feature_flag import FeatureFlag
+from products.notifications.backend.facade.api import NotificationType, SourceType, TargetType
+from products.notifications.backend.facade.testing import stored_notifications_for_team
+from products.product_tours.backend.models import ProductTour
+from products.surveys.backend.models import Survey
+
+NOTICE_TEAM_ID = 42
+SURVEY_ID = "0b1c5a52-6d1e-4c43-9a0e-5f2b8f8d1a01"
+TOUR_ID = "6f0d2c11-3b7a-4e58-8c1d-2a9e4b7f3c02"
+BETA_TESTERS = DeletedCohort(id=7, name="Beta testers", missing=False)
+
+
+def _affected_flag(flag_id: int, key: str, owner: FlagOwner | None = None) -> AffectedFlag:
+ return AffectedFlag(
+ flag_id=flag_id,
+ team_id=NOTICE_TEAM_ID,
+ key=key,
+ last_called_at=None,
+ cohorts=(BETA_TESTERS,),
+ owner=owner,
+ )
+
+
+class TestBuildNotice(SimpleTestCase):
+ @parameterized.expand(
+ [
+ (
+ "one_plain_flag",
+ [_affected_flag(101, "checkout-redesign")],
+ {
+ "source_type": SourceType.FEATURE_FLAG,
+ "source_id": "101",
+ "resource_type": "feature_flag",
+ "resource_id": "101",
+ "title": "Feature flag targets a deleted cohort: checkout-redesign",
+ "body": (
+ "Flag 'checkout-redesign' targets the deleted cohort 'Beta testers'. The flag fails to "
+ "evaluate until you remove that condition or choose another cohort."
+ ),
+ },
+ [],
+ [],
+ ),
+ (
+ "one_survey_targeting_flag",
+ [
+ _affected_flag(
+ 102,
+ "survey-targeting-onboarding",
+ FlagOwner(kind="survey", id=SURVEY_ID, name="Onboarding feedback"),
+ )
+ ],
+ {
+ "source_type": SourceType.SURVEY,
+ "source_id": SURVEY_ID,
+ "resource_type": "survey",
+ "title": "Survey targets a deleted cohort: Onboarding feedback",
+ },
+ ["Survey 'Onboarding feedback' targets the deleted cohort 'Beta testers'."],
+ [],
+ ),
+ (
+ "one_product_tour_flag",
+ [
+ _affected_flag(
+ 103, "product-tour-welcome", FlagOwner(kind="product_tour", id=TOUR_ID, name="Welcome tour")
+ )
+ ],
+ {
+ "source_type": None,
+ "source_url": f"/product_tours/{TOUR_ID}",
+ "resource_type": "feature_flag",
+ "title": "Product tour targets a deleted cohort: Welcome tour",
+ },
+ [],
+ [],
+ ),
+ (
+ "two_survey_flags",
+ [
+ _affected_flag(104, "survey-targeting-a", FlagOwner(kind="survey", id=SURVEY_ID, name="NPS")),
+ _affected_flag(105, "survey-targeting-b", FlagOwner(kind="survey", id=SURVEY_ID, name="CSAT")),
+ ],
+ {
+ "source_type": None,
+ "source_url": "/surveys",
+ "resource_type": "survey",
+ "title": "2 surveys target deleted cohorts",
+ },
+ ["survey 'NPS' (cohort 'Beta testers')"],
+ [],
+ ),
+ (
+ "six_plain_flags",
+ [_affected_flag(110 + n, f"rollout-{n}") for n in range(1, 7)],
+ {
+ "source_type": None,
+ "source_id": None,
+ "source_url": "/feature_flags",
+ "title": "6 feature flags target deleted cohorts",
+ },
+ ["flag 'rollout-1' (cohort 'Beta testers')", "and 1 more"],
+ ["'rollout-6'"],
+ ),
+ ]
+ )
+ def test_build_notice_links_and_words_the_notice_for_what_it_lists(
+ self,
+ _name: str,
+ flags: list[AffectedFlag],
+ expected_fields: dict[str, Any],
+ body_includes: list[str],
+ body_excludes: list[str],
+ ) -> None:
+ expected = {
+ "team_id": NOTICE_TEAM_ID,
+ "notification_type": NotificationType.FEATURE_FLAG_FAILURE,
+ "target_type": TargetType.TEAM,
+ "target_id": str(NOTICE_TEAM_ID),
+ "idempotency_key": "flags-deleted-cohort-notice",
+ **expected_fields,
+ }
+
+ notice = build_notice(NOTICE_TEAM_ID, flags)
+
+ assert {field: getattr(notice, field) for field in expected} == expected
+ assert [fragment for fragment in body_includes if fragment not in notice.body] == []
+ assert [fragment for fragment in body_excludes if fragment in notice.body] == []
+
+
+class TestNotifyFlagsOnDeletedCohorts(BaseTest):
+ def _run(self, *args: str) -> str:
+ out = StringIO()
+ call_command("notify_flags_on_deleted_cohorts", *args, stdout=out)
+ return out.getvalue()
+
+ def _cohort(self, name: str, *, team: Team | None = None, deleted: bool = False) -> Cohort:
+ return Cohort.objects.create(team=team or self.team, name=name, deleted=deleted)
+
+ def _flag(self, key: str, *cohort_ids: int, team: Team | None = None, **fields: Any) -> FeatureFlag:
+ return FeatureFlag.objects.create(
+ team=team or self.team,
+ created_by=self.user,
+ key=key,
+ filters={
+ "groups": [
+ {
+ "properties": [{"key": "id", "type": "cohort", "value": cohort_id} for cohort_id in cohort_ids],
+ "rollout_percentage": 100,
+ }
+ ]
+ },
+ **fields,
+ )
+
+ def test_finder_reports_active_flags_on_deleted_or_foreign_cohorts_only(self) -> None:
+ deleted_cohort = self._cohort("Beta testers", deleted=True)
+ live_cohort = self._cohort("Power users")
+ other_project_cohort = self._cohort("Early adopters", team=Team.objects.create(organization=self.organization))
+ nonexistent_cohort_id = other_project_cohort.id + 1000
+ on_deleted_cohort = self._flag("checkout-redesign", live_cohort.id, deleted_cohort.id)
+ self._flag("pricing-page-v2", live_cohort.id)
+ on_other_project_cohort = self._flag("onboarding-checklist", other_project_cohort.id, nonexistent_cohort_id)
+ self._flag("dark-mode", deleted_cohort.id, active=False)
+ self._flag("legacy-search", deleted_cohort.id, deleted=True)
+ survey = Survey.objects.create(
+ team=self.team, name="Onboarding feedback", type="popover", targeting_flag=on_deleted_cohort
+ )
+ tour = ProductTour.objects.create(
+ team=self.team, name="Welcome tour", internal_targeting_flag=on_other_project_cohort
+ )
+
+ result = find_flags_on_deleted_cohorts([self.team.id])
+
+ # The notice goes to this project, so it must not name a cohort that another project owns.
+ assert {team_id: sorted(flags, key=attrgetter("flag_id")) for team_id, flags in result.items()} == {
+ self.team.id: [
+ AffectedFlag(
+ flag_id=on_deleted_cohort.id,
+ team_id=self.team.id,
+ key="checkout-redesign",
+ last_called_at=None,
+ cohorts=(DeletedCohort(id=deleted_cohort.id, name="Beta testers", missing=False),),
+ owner=FlagOwner(kind="survey", id=str(survey.id), name="Onboarding feedback"),
+ ),
+ AffectedFlag(
+ flag_id=on_other_project_cohort.id,
+ team_id=self.team.id,
+ key="onboarding-checklist",
+ last_called_at=None,
+ cohorts=(
+ DeletedCohort(id=other_project_cohort.id, name=None, missing=True),
+ DeletedCohort(id=nonexistent_cohort_id, name=None, missing=True),
+ ),
+ owner=FlagOwner(kind="product_tour", id=str(tour.id), name="Welcome tour"),
+ ),
+ ]
+ }
+
+ @parameterized.expand(
+ [
+ (
+ "notifications_on",
+ True,
+ "Would deliver to up to 1 person.",
+ ["Feature flag targets a deleted cohort: checkout-redesign"],
+ "Already sent notice",
+ ),
+ (
+ "notifications_off",
+ False,
+ "Would not deliver: real-time notifications are off for this organization.",
+ [],
+ "Not delivered",
+ ),
+ ]
+ )
+ def test_dry_run_sends_nothing_and_send_notifies_only_the_listed_team_once(
+ self,
+ _name: str,
+ notifications_on: bool,
+ expected_delivery_line: str,
+ expected_titles: list[str],
+ expected_resend_line: str,
+ ) -> None:
+ self.enterContext(
+ patch(
+ "posthoganalytics.feature_enabled",
+ side_effect=lambda flag, *args, **kwargs: notifications_on and flag == "real-time-notifications",
+ )
+ )
+ self._flag("checkout-redesign", self._cohort("Beta testers", deleted=True).id)
+ other_team = Team.objects.create(organization=self.organization)
+ self._flag("search-v2", self._cohort("Old plan", team=other_team, deleted=True).id, team=other_team)
+
+ dry_run = self._run("--team-id", str(self.team.id))
+ with self.assertRaises(CommandError):
+ self._run("--send")
+
+ assert stored_notifications_for_team(self.team.id) == []
+ assert f"Team {self.team.id} ({self.team.name})" in dry_run
+ assert expected_delivery_line in dry_run
+
+ first_send = self._run("--send", "--team-id", str(self.team.id))
+
+ sent = stored_notifications_for_team(self.team.id)
+ assert [notice.title for notice in sent] == expected_titles
+ assert stored_notifications_for_team(other_team.id) == []
+ if not notifications_on:
+ assert "Not delivered" in first_send
+
+ second_send = self._run("--send", "--team-id", str(self.team.id))
+
+ assert stored_notifications_for_team(self.team.id) == sent
+ assert expected_resend_line in second_send
+
+ @parameterized.expand(
+ [
+ ("denied_the_listed_flag", "feature_flag", True),
+ ("denied_all_surveys", "survey", False),
+ ]
+ )
+ def test_send_leaves_out_a_member_denied_an_object_the_notice_names(
+ self, _name: str, denied_resource: str, deny_only_the_flag: bool
+ ) -> None:
+ self.enterContext(
+ patch(
+ "posthoganalytics.feature_enabled",
+ side_effect=lambda flag, *args, **kwargs: flag == "real-time-notifications",
+ )
+ )
+ self.organization.available_product_features = [
+ {"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}
+ ]
+ self.organization.save()
+ deleted_cohort = self._cohort("Beta testers", deleted=True)
+ plain_flag = self._flag("checkout-redesign", deleted_cohort.id)
+ survey_flag = self._flag("survey-targeting-nps", deleted_cohort.id)
+ Survey.objects.create(team=self.team, name="NPS check-in", type="popover", targeting_flag=survey_flag)
+ denied = User.objects.create_and_join(self.organization, "[email protected]", "password")
+ AccessControl.objects.create(
+ team=self.team,
+ resource=denied_resource,
+ resource_id=str(plain_flag.id) if deny_only_the_flag else None,
+ organization_member=OrganizationMembership.objects.get(organization=self.organization, user=denied),
+ access_level="none",
+ )
+
+ self._run("--send", "--team-id", str(self.team.id))
+
+ assert [notice.resolved_user_ids for notice in stored_notifications_for_team(self.team.id)] == [[self.user.id]]
diff --git a/products/notifications/backend/facade/enums.py b/products/notifications/backend/facade/enums.py
index d68a14c001fe..1f6248c983b1 100644
--- a/products/notifications/backend/facade/enums.py
+++ b/products/notifications/backend/facade/enums.py
@@ -23,6 +23,7 @@ class NotificationType(str, Enum):
NOTIFICATION_SETTINGS_CHANGED = "notification_settings_changed"
DATA_QUALITY_CHECK_FAILURE = "data_quality_check_failure"
WORKFLOW_SUGGESTION = "workflow_suggestion"
+ FEATURE_FLAG_FAILURE = "feature_flag_failure"
class Priority(str, Enum):
diff --git a/tach.toml b/tach.toml
index 40250e39dd27..c6e17f82e885 100644
--- a/tach.toml
+++ b/tach.toml
@@ -536,6 +536,7 @@ depends_on = [
"products.product_analytics",
"products.product_tours",
"products.surveys", "products.access_control",
+ "products.notifications",
]
layer = "modules"