Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): enroll a percentage of self-driving orgs in anomaly scoring (#125622)

posthog[bot]6542fd370c0d
diff --git a/products/signals/backend/anomaly_scores/config.py b/products/signals/backend/anomaly_scores/config.py
index 5562961c231b..d91b06987e6e 100644
--- a/products/signals/backend/anomaly_scores/config.py
+++ b/products/signals/backend/anomaly_scores/config.py
@@ -5,6 +5,7 @@ field falls back to its default, and no default is unbounded.
"""
import uuid
+import hashlib
import dataclasses
from enum import StrEnum
from typing import TYPE_CHECKING, Any
@@ -118,8 +119,13 @@ DEFAULT_TICK_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONT
class AnomalyScoringConfig:
enabled: bool = False
organization_ids: tuple[str, ...] = ()
- # Empty means every team of the organizations in `organization_ids`.
+ # Empty means every team of the organizations in `organization_ids`. The percentage rollout ignores it.
allow_team_ids: tuple[int, ...] = ()
+ # Scores this percentage of the organizations with a project that has an enabled signal source.
+ # Only those projects are scored. A higher percentage keeps the organizations of a lower one.
+ self_driving_organization_percentage: float = 0.0
+ # The tick never scores these organizations, even when another field enrolls them.
+ deny_organization_ids: tuple[str, ...] = ()
skip_team_ids: tuple[int, ...] = ()
intervals: tuple[IntervalType, ...] = DEFAULT_TICK_INTERVALS
# Smooths the spend over the hour, so one tick cannot use the whole query budget. With the split on,
@@ -166,25 +172,32 @@ def _positive_int(value: Any, default: int) -> int:
return value if isinstance(value, int) and not isinstance(value, bool) and value > 0 else default
+def _percentage(value: Any, default: float) -> float:
+ if isinstance(value, bool) or not isinstance(value, int | float) or not 0 <= value <= 100:
+ return default
+ return float(value)
+
+
def _team_ids(value: Any) -> tuple[int, ...]:
if not isinstance(value, list):
return ()
return tuple(item for item in value if isinstance(item, int) and not isinstance(item, bool))
-def _is_uuid(value: str) -> bool:
+def _canonical_uuid(value: str) -> str | None:
try:
- uuid.UUID(value)
+ return str(uuid.UUID(value))
except ValueError:
- return False
- return True
+ return None
def _organization_ids(value: Any) -> tuple[str, ...]:
if not isinstance(value, list):
return ()
- # A value that is not a UUID would fail the organization lookup of the whole tick.
- return tuple(item for item in value if isinstance(item, str) and _is_uuid(item))
+ # A value that is not a UUID would fail the organization lookup of the whole tick. The canonical form
+ # lets `covers_team` compare ids as strings, so an id in another UUID format still matches.
+ ids = (_canonical_uuid(item) for item in value if isinstance(item, str))
+ return tuple(item for item in ids if item is not None)
def _intervals(value: Any) -> tuple[IntervalType, ...]:
@@ -230,6 +243,10 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
enabled=True,
organization_ids=_organization_ids(payload.get("organization_ids")),
allow_team_ids=_team_ids(payload.get("allow_team_ids")),
+ self_driving_organization_percentage=_percentage(
+ payload.get("self_driving_organization_percentage"), defaults.self_driving_organization_percentage
+ ),
+ deny_organization_ids=_organization_ids(payload.get("deny_organization_ids")),
skip_team_ids=_team_ids(payload.get("skip_team_ids")),
intervals=_intervals(payload.get("intervals")),
max_insights_per_tick=min(
@@ -279,15 +296,41 @@ def parse_config(payload: Any) -> AnomalyScoringConfig:
)
+# A new salt reorders every organization, so change the version only to start a new rollout order.
+_ROLLOUT_SALT = "anomaly-scoring-self-driving-v1"
+_ROLLOUT_BUCKETS = 10_000
+
+
+def organization_rollout_bucket(organization_id: "uuid.UUID | str") -> int:
+ """The organization's fixed place in the percentage rollout, from 0 to `_ROLLOUT_BUCKETS` - 1."""
+ digest = hashlib.sha256(f"{_ROLLOUT_SALT}:{uuid.UUID(str(organization_id))}".encode()).digest()
+ return int.from_bytes(digest[:8], "big") % _ROLLOUT_BUCKETS
+
+
+def in_rollout(config: AnomalyScoringConfig, organization_id: "uuid.UUID | str") -> bool:
+ threshold = config.self_driving_organization_percentage * _ROLLOUT_BUCKETS / 100
+ return organization_rollout_bucket(organization_id) < threshold
+
+
def covers_team(config: AnomalyScoringConfig, team: "Team") -> bool:
- """Whether the tick scores this team's insights. The event quota check of `queue.eligible_teams` is left out."""
- return (
- config.enabled
- and str(team.organization_id) in config.organization_ids
- and team.id not in config.skip_team_ids
- and (not config.allow_team_ids or team.id in config.allow_team_ids)
+ """Whether the tick scores this team's insights. The single-team form of `queue.eligible_teams`.
+
+ The event quota check of `queue.eligible_teams` is left out. The skip list and the denylist win over every
+ enrollment. The percentage rollout enrolls only the projects that have an enabled signal source.
+ """
+ organization_id = str(team.organization_id)
+ if not config.enabled or team.id in config.skip_team_ids or organization_id in config.deny_organization_ids:
+ return False
+ if organization_id in config.organization_ids:
+ return not config.allow_team_ids or team.id in config.allow_team_ids
+ if not in_rollout(config, organization_id):
+ return False
+ from products.signals.backend.facade.api import (
+ has_enabled_source, # noqa: PLC0415 — the facade imports this module through `search`
)
+ return has_enabled_source(team.id)
+
def read_payload() -> dict | None:
return read_flag_payload(ANOMALY_SCORING_FLAG, ANOMALY_SCORING_FLAG_DISTINCT_ID)
@@ -300,5 +343,7 @@ def read_config(payload: Any) -> AnomalyScoringConfig:
enabled=config.enabled,
organizations=len(config.organization_ids),
allow_teams=len(config.allow_team_ids),
+ self_driving_organization_percentage=config.self_driving_organization_percentage,
+ denied_organizations=len(config.deny_organization_ids),
)
return config
diff --git a/products/signals/backend/anomaly_scores/queue.py b/products/signals/backend/anomaly_scores/queue.py
index d3b618580f04..5e11dec55ed0 100644
--- a/products/signals/backend/anomaly_scores/queue.py
+++ b/products/signals/backend/anomaly_scores/queue.py
@@ -51,7 +51,7 @@ from products.product_analytics.backend.facade.api import (
insights_including_soft_deleted_for_team,
saved_trends_insights,
)
-from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringRun
+from products.signals.backend.anomaly_scores.config import AnomalyScoringConfig, ScoringRun, in_rollout
from products.signals.backend.anomaly_scores.scoring import bucket_end
from products.signals.backend.anomaly_scores.selection import (
Candidate,
@@ -179,6 +179,9 @@ class Selection:
active_reasons: dict[str, int] = dataclasses.field(default_factory=dict)
# One entry for each eligible team, also a team with no active insight.
census: list[TeamCensus] = dataclasses.field(default_factory=list)
+ # Eligible teams, and their organizations.
+ enrolled_teams: int = 0
+ enrolled_organizations: int = 0
@frozen
@@ -200,18 +203,32 @@ class QueryBudget:
def eligible_teams(config: AnomalyScoringConfig) -> list[Team]:
- if not config.enabled or not config.organization_ids:
+ """The teams that `config.covers_team` enrolls, in a fixed number of queries, less the teams over their event quota."""
+ if not config.enabled:
return []
teams = (
- Team.objects.filter(organization_id__in=config.organization_ids)
- .exclude(id__in=config.skip_team_ids)
+ Team.objects.exclude(id__in=config.skip_team_ids)
+ .exclude(organization_id__in=config.deny_organization_ids)
.exclude(id__in=SignalTeamConfig.objects.filter(anomaly_scoring_enabled=False).values("team_id"))
)
- if config.allow_team_ids:
- teams = teams.filter(id__in=config.allow_team_ids)
+ enrolled: set[int] = set()
+ if config.organization_ids:
+ listed = teams.filter(organization_id__in=config.organization_ids)
+ if config.allow_team_ids:
+ listed = listed.filter(id__in=config.allow_team_ids)
+ enrolled.update(listed.values_list("id", flat=True))
+ if config.self_driving_organization_percentage > 0:
+ from products.signals.backend.facade.api import (
+ team_ids_with_enabled_source, # noqa: PLC0415 — the facade imports this module through `search`
+ )
+
+ with_source = teams.filter(id__in=team_ids_with_enabled_source()).values_list("id", "organization_id")
+ enrolled.update(team_id for team_id, organization_id in with_source if in_rollout(config, organization_id))
+ if not enrolled:
+ return []
# Capture drops every event of a project over its event quota, so scoring it wastes the query.
limited = set(list_limited_team_attributes(QuotaResource.EVENTS, QuotaLimitingCaches.QUOTA_LIMITER_CACHE_KEY))
- return [team for team in teams if team.api_token not in limited]
+ return [team for team in Team.objects.filter(id__in=enrolled) if team.api_token not in limited]
def turned_off_in_settings(team: Team) -> bool:
@@ -662,6 +679,8 @@ def select_insights(
active=active,
active_reasons=active_reasons,
census=census,
+ enrolled_teams=len(teams),
+ enrolled_organizations=len({team.organization_id for team in teams.values()}),
)
diff --git a/products/signals/backend/anomaly_scores/tick.py b/products/signals/backend/anomaly_scores/tick.py
index aa780fb60fe1..25f67d3a9385 100644
--- a/products/signals/backend/anomaly_scores/tick.py
+++ b/products/signals/backend/anomaly_scores/tick.py
@@ -122,6 +122,8 @@ class AnomalyScoringTickPlan:
train_backlog_hard: int = 0
active: int = 0
active_reasons: dict[str, int] | None = None
+ enrolled_teams: int = 0
+ enrolled_organizations: int = 0
@frozen
@@ -170,6 +172,8 @@ class AnomalyScoringTickResult:
# Active insights of the eligible teams, and how many of them have each activity reason.
active: int = 0
active_reasons: dict[str, int] | None = None
+ enrolled_teams: int = 0
+ enrolled_organizations: int = 0
def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPlan:
@@ -195,6 +199,8 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> AnomalyScoringTickPl
train_backlog_hard=selection.train_backlog_hard,
active=selection.active,
active_reasons=selection.active_reasons,
+ enrolled_teams=selection.enrolled_teams,
+ enrolled_organizations=selection.enrolled_organizations,
)
@@ -423,6 +429,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
train_backlog_hard=result.train_backlog_hard,
active=result.active,
active_reasons=result.active_reasons,
+ enrolled_teams=result.enrolled_teams,
+ enrolled_organizations=result.enrolled_organizations,
)
ph_background_capture()(
distinct_id=ANOMALY_SCORING_FLAG_DISTINCT_ID,
@@ -448,6 +456,8 @@ def capture_tick(result: AnomalyScoringTickResult) -> None:
"train_backlog_hard": result.train_backlog_hard,
"active_insights": result.active,
**{f"active_{reason}": count for reason, count in (result.active_reasons or {}).items()},
+ "enrolled_teams": result.enrolled_teams,
+ "enrolled_organizations": result.enrolled_organizations,
},
)
@@ -545,6 +555,8 @@ class AnomalyScoringTickWorkflow:
train_backlog_hard=plan.train_backlog_hard,
active=plan.active,
active_reasons=plan.active_reasons,
+ enrolled_teams=plan.enrolled_teams,
+ enrolled_organizations=plan.enrolled_organizations,
)
await workflow.execute_activity(
capture_anomaly_scoring_tick_activity,
diff --git a/products/signals/backend/test/test_anomaly_scoring_tick.py b/products/signals/backend/test/test_anomaly_scoring_tick.py
index 058f03159b35..3b90a87820b8 100644
--- a/products/signals/backend/test/test_anomaly_scoring_tick.py
+++ b/products/signals/backend/test/test_anomaly_scoring_tick.py
@@ -38,6 +38,7 @@ from products.signals.backend.anomaly_scores.config import (
DISABLED,
AnomalyScoringConfig,
ScoringRun,
+ covers_team,
parse_config,
)
from products.signals.backend.anomaly_scores.emission import SCORER_DISTINCT_ID, score_events
@@ -84,7 +85,7 @@ from products.signals.backend.anomaly_scores.tick import (
plan_tick,
run_insight,
)
-from products.signals.backend.models import AnomalyScoringState, SignalTeamConfig
+from products.signals.backend.models import AnomalyScoringState, SignalSourceConfig, SignalTeamConfig
from products.subscriptions.backend.facade.testing import create_subscription
NEW_YORK = ZoneInfo("America/New_York")
@@ -141,6 +142,8 @@ class TestTickConfig(SimpleTestCase):
"enabled": True,
"organization_ids": [ORGANIZATION_ID, "not-a-uuid"],
"allow_team_ids": [2, True, "3"],
+ "self_driving_organization_percentage": 101,
+ "deny_organization_ids": [ORGANIZATION_ID.upper(), "not-a-uuid", 3],
"intervals": ["minute", "week"],
"max_insights_per_tick": 0,
"limits": {"max_backfill_buckets": 100_000, "max_series": -1},
@@ -152,6 +155,8 @@ class TestTickConfig(SimpleTestCase):
assert config.organization_ids == (ORGANIZATION_ID,)
assert config.allow_team_ids == (2,)
+ assert config.self_driving_organization_percentage == 0.0
+ assert config.deny_organization_ids == (ORGANIZATION_ID,)
assert config.intervals == (IntervalType.WEEK,)
assert config.max_insights_per_tick == DISABLED.max_insights_per_tick
assert (config.limits.max_backfill_buckets, config.limits.max_series) == (90, 20)
@@ -733,6 +738,62 @@ class TestEligibleTeams(BaseTest):
assert eligible == {teams[name].id for name in expected}
+ @parameterized.expand(
+ [
+ # name, listed, percentage, denied, skipped, allow_other, has_source, enrolled
+ ("listed_without_source", True, 0.0, False, False, False, False, True),
+ ("listed_and_denied", True, 0.0, True, False, False, False, False),
+ ("rollout", False, 100.0, False, False, False, True, True),
+ ("rollout_without_source", False, 100.0, False, False, False, False, False),
+ ("rollout_at_zero", False, 0.0, False, False, False, True, False),
+ ("rollout_denied", False, 100.0, True, False, False, True, False),
+ ("rollout_team_skipped", False, 100.0, False, True, False, True, False),
+ ("rollout_ignores_the_allowlist", False, 100.0, False, False, True, True, True),
+ ]
+ )
+ def test_enrollment_by_list_and_rollout(
+ self,
+ _name: str,
+ listed: bool,
+ percentage: float,
+ denied: bool,
+ skipped: bool,
+ allow_other: bool,
+ has_source: bool,
+ enrolled: bool,
+ ) -> None:
+ other = Team.objects.create(organization=Organization.objects.create(name="other"), name="other")
+ own_organization = (str(self.organization.id),)
+ config = AnomalyScoringConfig(
+ enabled=True,
+ organization_ids=own_organization if listed else (),
+ self_driving_organization_percentage=percentage,
+ deny_organization_ids=own_organization if denied else (),
+ skip_team_ids=(self.team.id,) if skipped else (),
+ allow_team_ids=(other.id,) if allow_other else (),
+ )
+ if has_source:
+ SignalSourceConfig.objects.create(
+ team=self.team, source_product="error_tracking", source_type="issue_created", enabled=True
+ )
+
+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
+ eligible = {team.id for team in eligible_teams(config)}
+
+ assert covers_team(config, Team.objects.get(id=self.team.id)) is enrolled
+ assert (self.team.id in eligible) is enrolled
+ assert other.id not in eligible
+
+ def test_rollout_skips_a_team_that_turned_scoring_off(self) -> None:
+ SignalSourceConfig.objects.create(
+ team=self.team, source_product="error_tracking", source_type="issue_created", enabled=True
+ )
+ SignalTeamConfig.objects.create(team=self.team, anomaly_scoring_enabled=False)
+ config = AnomalyScoringConfig(enabled=True, self_driving_organization_percentage=100.0)
+
+ with patch("products.signals.backend.anomaly_scores.queue.list_limited_team_attributes", return_value=[]):
+ assert self.team.id not in {team.id for team in eligible_teams(config)}
+
_DEFAULT_INTERVALS = (IntervalType.DAY, IntervalType.WEEK, IntervalType.MONTH)
_LONG_AGO = NOW - timedelta(days=91)