:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): enroll a percentage of self-driving orgs in anomaly scoring (#125622)
posthog[bot]6542fd370c0d
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.pyindex 5562961c231b..d91b06987e6e 100644--- a/products/signals/backend/anomaly_scores/config.py+++ b/products/signals/backend/anomaly_scores/config.py@@ -5,6 +5,7 @@ field falls back to its default, and no default is unbounded."""import uuid+import hashlibimport dataclassesfrom enum import StrEnumfrom typing import TYPE_CHECKING, Any@@ -118,8 +119,13 @@ DEFAULT_TICK_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONTclass AnomalyScoringConfig:enabled: bool = Falseorganization_ids: tuple[str, ...] = ()- # Empty means every team of the organizations in `organization_ids`.+ # Empty means every team of the organizations in `organization_ids`. The percentage rollout ignores it.allow_team_ids: tuple[int, ...] = ()+ # Scores this percentage of the organizations with a project that has an enabled signal source.+ # Only those projects are scored. A higher percentage keeps the organizations of a lower one.+ self_driving_organization_percentage: float = 0.0+ # The tick never scores these organizations, even when another field enrolls them.+ deny_organization_ids: tuple[str, ...] = ()skip_team_ids: tuple[int, ...] = ()intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS# Smooths the spend over the hour, so one tick cannot use the whole query budget. With the split on,@@ -166,25 +172,32 @@ def _positive_int(value: Any, default: int) -> int:return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else default+def _percentage(value: Any, default: float) -> float:+ if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value <= 100:+ return default+ return float(value)++def _team_ids(value: Any) -> tuple[int, ...]:if not isinstance(value, list):return ()return tuple(item for item in value if isinstance(item, int) and not isinstance(item, bool))-def _is_uuid(value: str) -> bool:+def _canonical_uuid(value: str) -> str | None:try:- uuid.UUID(value)+ return str(uuid.UUID(value))except ValueError:- return False- return True+ return Nonedef _organization_ids(value: Any) -> tuple[str, ...]:if not isinstance(value, list):return ()- # A value that is not a UUID would fail the organization lookup of the whole tick.- return tuple(item for item in value if isinstance(item, str) and _is_uuid(item))+ # A value that is not a UUID would fail the organization lookup of the whole tick. The canonical form+ # lets `covers_team` compare ids as strings, so an id in another UUID format still matches.+ ids = (_canonical_uuid(item) for item in value if isinstance(item, str))+ return tuple(item for item in ids if item is not None)def _intervals(value: Any) -> tuple[IntervalType, ...]:@@ -230,6 +243,10 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:enabled=True,organization_ids=_organization_ids(payload.get("organization_ids")),allow_team_ids=_team_ids(payload.get("allow_team_ids")),+ self_driving_organization_percentage=_percentage(+ payload.get("self_driving_organization_percentage"), defaults.self_driving_organization_percentage+ ),+ deny_organization_ids=_organization_ids(payload.get("deny_organization_ids")),skip_team_ids=_team_ids(payload.get("skip_team_ids")),intervals=_intervals(payload.get("intervals")),max_insights_per_tick=min(@@ -279,15 +296,41 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:)+# A new salt reorders every organization, so change the version only to start a new rollout order.+_ROLLOUT_SALT = "anomaly-scoring-self-driving-v1"+_ROLLOUT_BUCKETS = 10_000+++def organization_rollout_bucket(organization_id: "uuid.UUID | str") -> int:+ """The organization's fixed place in the percentage rollout, from 0 to `_ROLLOUT_BUCKETS` - 1."""+ digest = hashlib.sha256(f"{_ROLLOUT_SALT}:{uuid.UUID(str(organization_id))}".encode()).digest()+ return int.from_bytes(digest[:8], "big") % _ROLLOUT_BUCKETS+++def in_rollout(config: AnomalyScoringConfig, organization_id: "uuid.UUID | str") -> bool:+ threshold = config.self_driving_organization_percentage * _ROLLOUT_BUCKETS / 100+ return organization_rollout_bucket(organization_id) < threshold++def covers_team(config: AnomalyScoringConfig, team: "Team") -> bool:- """Whether the tick scores this team's insights. The event quota check of `queue.eligible_teams` is left out."""- return (- config.enabled- and str(team.organization_id) in config.organization_ids- and team.id not in config.skip_team_ids- and (not config.allow_team_ids or team.id in config.allow_team_ids)+ """Whether the tick scores this team's insights. The single-team form of `queue.eligible_teams`.++ The event quota check of `queue.eligible_teams` is left out. The skip list and the denylist win over every+ enrollment. The percentage rollout enrolls only the projects that have an enabled signal source.+ """+ organization_id = str(team.organization_id)+ if not config.enabled or team.id in config.skip_team_ids or organization_id in config.deny_organization_ids:+ return False+ if organization_id in config.organization_ids:+ return not config.allow_team_ids or team.id in config.allow_team_ids+ if not in_rollout(config, organization_id):+ return False+ from products.signals.backend.facade.api import (+ has_enabled_source, # noqa: PLC0415 — the facade imports this module through `search`)+ return has_enabled_source(team.id)+def read_payload() -> dict | None:return read_flag_payload(ANOMALY_SCORING_FLAG, ANOMALY_SCORING_FLAG_DISTINCT_ID)@@ -300,5 +343,7 @@ def read_config(payload: Any) -> AnomalyScoringConfig:enabled=config.enabled,organizations=len(config.organization_ids),allow_teams=len(config.allow_team_ids),+ self_driving_organization_percentage=config.self_driving_organization_percentage,+ denied_organizations=len(config.deny_organization_ids),)return configdiff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.pyindex d3b618580f04..5e11dec55ed0 100644--- a/products/signals/backend/anomaly_scores/queue.py+++ b/products/signals/backend/anomaly_scores/queue.py@@ -51,7 +51,7 @@ from products.product_analytics.backend.facade.api import (insights_including_soft_deleted_for_team,saved_trends_insights,)-from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringRun+from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringRun, in_rolloutfrom products.signals.backend.anomaly_scores.scoring import bucket_endfrom products.signals.backend.anomaly_scores.selection import (Candidate,@@ -179,6 +179,9 @@ class Selection:active_reasons: dict[str, int] = dataclasses.field(default_factory=dict)# One entry for each eligible team, also a team with no active insight.census: list[TeamCensus] = dataclasses.field(default_factory=list)+ # Eligible teams, and their organizations.+ enrolled_teams: int = 0+ enrolled_organizations: int = 0@frozen@@ -200,18 +203,32 @@ class QueryBudget:def eligible_teams(config: AnomalyScoringConfig) -> list[Team]:- if not config.enabled or not config.organization_ids:+ """The teams that `config.covers_team` enrolls, in a fixed number of queries, less the teams over their event quota."""+ if not config.enabled:return []teams = (- Team.objects.filter(organization_id__in=config.organization_ids)- .exclude(id__in=config.skip_team_ids)+ Team.objects.exclude(id__in=config.skip_team_ids)+ .exclude(organization_id__in=config.deny_organization_ids).exclude(id__in=SignalTeamConfig.objects.filter(anomaly_scoring_enabled=False).values("team_id")))- if config.allow_team_ids:- teams = teams.filter(id__in=config.allow_team_ids)+ enrolled: set[int] = set()+ if config.organization_ids:+ listed = teams.filter(organization_id__in=config.organization_ids)+ if config.allow_team_ids:+ listed = listed.filter(id__in=config.allow_team_ids)+ enrolled.update(listed.values_list("id", flat=True))+ if config.self_driving_organization_percentage > 0:+ from products.signals.backend.facade.api import (+ team_ids_with_enabled_source, # noqa: PLC0415 — the facade imports this module through `search`+ )++ with_source = teams.filter(id__in=team_ids_with_enabled_source()).values_list("id", "organization_id")+ enrolled.update(team_id for team_id, organization_id in with_source if in_rollout(config, organization_id))+ if not enrolled:+ return []# Capture drops every event of a project over its event quota, so scoring it wastes the query.limited = set(list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY))- return [team for team in teams if team.api_token not in limited]+ return [team for team in Team.objects.filter(id__in=enrolled) if team.api_token not in limited]def turned_off_in_settings(team: Team) -> bool:@@ -662,6 +679,8 @@ def select_insights(active=active,active_reasons=active_reasons,census=census,+ enrolled_teams=len(teams),+ enrolled_organizations=len({team.organization_id for team in teams.values()}),)diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.pyindex aa780fb60fe1..25f67d3a9385 100644--- a/products/signals/backend/anomaly_scores/tick.py+++ b/products/signals/backend/anomaly_scores/tick.py@@ -122,6 +122,8 @@ class AnomalyScoringTickPlan:train_backlog_hard: int = 0active: int = 0active_reasons: dict[str, int] | None = None+ enrolled_teams: int = 0+ enrolled_organizations: int = 0@frozen@@ -170,6 +172,8 @@ class AnomalyScoringTickResult:# Active insights of the eligible teams, and how many of them have each activity reason.active: int = 0active_reasons: dict[str, int] | None = None+ enrolled_teams: int = 0+ enrolled_organizations: int = 0def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:@@ -195,6 +199,8 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPltrain_backlog_hard=selection.train_backlog_hard,active=selection.active,active_reasons=selection.active_reasons,+ enrolled_teams=selection.enrolled_teams,+ enrolled_organizations=selection.enrolled_organizations,)@@ -423,6 +429,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:train_backlog_hard=result.train_backlog_hard,active=result.active,active_reasons=result.active_reasons,+ enrolled_teams=result.enrolled_teams,+ enrolled_organizations=result.enrolled_organizations,)ph_background_capture()(distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,@@ -448,6 +456,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:"train_backlog_hard": result.train_backlog_hard,"active_insights": result.active,**{f"active_{reason}": count for reason, count in (result.active_reasons or {}).items()},+ "enrolled_teams": result.enrolled_teams,+ "enrolled_organizations": result.enrolled_organizations,},)@@ -545,6 +555,8 @@ class AnomalyScoringTickWorkflow:train_backlog_hard=plan.train_backlog_hard,active=plan.active,active_reasons=plan.active_reasons,+ enrolled_teams=plan.enrolled_teams,+ enrolled_organizations=plan.enrolled_organizations,)await workflow.execute_activity(capture_anomaly_scoring_tick_activity,diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.pyindex 058f03159b35..3b90a87820b8 100644--- a/products/signals/backend/test/test_anomaly_scoring_tick.py+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py@@ -38,6 +38,7 @@ from products.signals.backend.anomaly_scores.config import (DISABLED,AnomalyScoringConfig,ScoringRun,+ covers_team,parse_config,)from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events@@ -84,7 +85,7 @@ from products.signals.backend.anomaly_scores.tick import (plan_tick,run_insight,)-from products.signals.backend.models import AnomalyScoringState, SignalTeamConfig+from products.signals.backend.models import AnomalyScoringState, SignalSourceConfig, SignalTeamConfigfrom products.subscriptions.backend.facade.testing import create_subscriptionNEW_YORK = ZoneInfo("America/New_York")@@ -141,6 +142,8 @@ class TestTickConfig(SimpleTestCase):"enabled": True,"organization_ids": [ORGANIZATION_ID, "not-a-uuid"],"allow_team_ids": [2, True, "3"],+ "self_driving_organization_percentage": 101,+ "deny_organization_ids": [ORGANIZATION_ID.upper(), "not-a-uuid", 3],"intervals": ["minute", "week"],"max_insights_per_tick": 0,"limits": {"max_backfill_buckets": 100_000, "max_series": -1},@@ -152,6 +155,8 @@ class TestTickConfig(SimpleTestCase):assert config.organization_ids == (ORGANIZATION_ID,)assert config.allow_team_ids == (2,)+ assert config.self_driving_organization_percentage == 0.0+ assert config.deny_organization_ids == (ORGANIZATION_ID,)assert config.intervals == (IntervalType.WEEK,)assert config.max_insights_per_tick == DISABLED.max_insights_per_tickassert (config.limits.max_backfill_buckets, config.limits.max_series) == (90, 20)@@ -733,6 +738,62 @@ class TestEligibleTeams(BaseTest):assert eligible == {teams[name].id for name in expected}+ @parameterized.expand(+ [+ # name, listed, percentage, denied, skipped, allow_other, has_source, enrolled+ ("listed_without_source", True, 0.0, False, False, False, False, True),+ ("listed_and_denied", True, 0.0, True, False, False, False, False),+ ("rollout", False, 100.0, False, False, False, True, True),+ ("rollout_without_source", False, 100.0, False, False, False, False, False),+ ("rollout_at_zero", False, 0.0, False, False, False, True, False),+ ("rollout_denied", False, 100.0, True, False, False, True, False),+ ("rollout_team_skipped", False, 100.0, False, True, False, True, False),+ ("rollout_ignores_the_allowlist", False, 100.0, False, False, True, True, True),+ ]+ )+ def test_enrollment_by_list_and_rollout(+ self,+ _name: str,+ listed: bool,+ percentage: float,+ denied: bool,+ skipped: bool,+ allow_other: bool,+ has_source: bool,+ enrolled: bool,+ ) -> None:+ other = Team.objects.create(organization=Organization.objects.create(name="other"), name="other")+ own_organization = (str(self.organization.id),)+ config = AnomalyScoringConfig(+ enabled=True,+ organization_ids=own_organization if listed else (),+ self_driving_organization_percentage=percentage,+ deny_organization_ids=own_organization if denied else (),+ skip_team_ids=(self.team.id,) if skipped else (),+ allow_team_ids=(other.id,) if allow_other else (),+ )+ if has_source:+ SignalSourceConfig.objects.create(+ team=self.team, source_product="error_tracking", source_type="issue_created", enabled=True+ )++ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):+ eligible = {team.id for team in eligible_teams(config)}++ assert covers_team(config, Team.objects.get(id=self.team.id)) is enrolled+ assert (self.team.id in eligible) is enrolled+ assert other.id not in eligible++ def test_rollout_skips_a_team_that_turned_scoring_off(self) -> None:+ SignalSourceConfig.objects.create(+ team=self.team, source_product="error_tracking", source_type="issue_created", enabled=True+ )+ SignalTeamConfig.objects.create(team=self.team, anomaly_scoring_enabled=False)+ config = AnomalyScoringConfig(enabled=True, self_driving_organization_percentage=100.0)++ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):+ assert self.team.id not in {team.id for team in eligible_teams(config)}+_DEFAULT_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONTH)_LONG_AGO = NOW - timedelta(days=91)