:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): count activity summary enrollment and builds, add a preview (#122728)
posthog[bot]946dea8b8fb0
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.pyindex 4fe64bc5d5b3..7cd019195e57 100644--- a/products/signals/backend/activity_summary/logic.py+++ b/products/signals/backend/activity_summary/logic.py@@ -79,6 +79,8 @@ class ActivitySummaryUpdate:inputs_used: dict[str, Any] | None = Noneinput_watermark: datetime.datetime | None = Nonerebuilt: bool = False+ # The pair had no generated summary before this update.+ new_summary: bool = Falseclass StoredSummaryVersion(Protocol):@@ -317,6 +319,7 @@ def update_activity_summary(inputs_used=inputs_used,input_watermark=watermark if outcome.summary is not None else None,rebuilt=rebuild,+ new_summary=stored is None or stored.generated_at is None,)if dry_run:return resultdiff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.pyindex 5d11e5f4b7bd..0391e46aa011 100644--- a/products/signals/backend/activity_summary/queue.py+++ b/products/signals/backend/activity_summary/queue.py@@ -177,32 +177,51 @@ def _in_rollout(config: ActivitySummaryConfig, organization_id: uuid.UUID) -> boreturn organization_rollout_bucket(organization_id) < threshold-def enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:+@frozen+class Enrollment:+ team_ids: list[int]+ organization_ids: list[uuid.UUID]+ # The organizations that a 100 percent rollout enrolls, with or without manual enrollment.+ qualifying_organization_ids: list[uuid.UUID]+++def enrollment(config: ActivitySummaryConfig) -> Enrollment:"""The teams that `is_team_enrolled_for_generation` enrolls, in a fixed number of queries."""if not config.enabled:- return []+ return Enrollment(team_ids=[], organization_ids=[], qualifying_organization_ids=[])+ from products.signals.backend.facade.api import (+ team_ids_with_enabled_source, # noqa: PLC0415 — the facade imports this module through `read`+ )+listed = _parse_organization_ids(config.organization_ids)teams = (Team.objects.filter(organization__is_ai_data_processing_approved=True).exclude(id__in=config.skip_team_ids).exclude(organization_id__in=_parse_organization_ids(config.deny_organization_ids)))+ # The teams with an enabled source come first, because they are far fewer than the approved teams.+ with_source: dict[int, uuid.UUID] = dict(+ teams.filter(id__in=team_ids_with_enabled_source()).values_list("id", "organization_id")+ )+ enrolled: dict[int, uuid.UUID]if config.enroll_all_approved:- return sorted(teams.values_list("id", flat=True))- team_ids = set(teams.filter(organization_id__in=listed).values_list("id", flat=True)) if listed else set()- if config.self_driving_organization_percentage > 0:- from products.signals.backend.facade.api import (- team_ids_with_enabled_source, # noqa: PLC0415 — the facade imports this module through `read`- )-- # The teams with an enabled source come first, because they are far fewer than the approved teams.- candidates = teams.filter(id__in=team_ids_with_enabled_source()).exclude(organization_id__in=listed)- team_ids |= {- team_id- for team_id, organization_id in candidates.values_list("id", "organization_id")+ enrolled = dict(teams.values_list("id", "organization_id"))+ else:+ enrolled = dict(teams.filter(organization_id__in=listed).values_list("id", "organization_id")) if listed else {}+ enrolled |= {+ team_id: organization_id+ for team_id, organization_id in with_source.items()if _in_rollout(config, organization_id)}- return sorted(team_ids)+ return Enrollment(+ team_ids=sorted(enrolled),+ organization_ids=sorted(set(enrolled.values())),+ qualifying_organization_ids=sorted(set(with_source.values())),+ )+++def enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:+ return enrollment(config).team_idsdef is_team_enrolled_for_generation(config: ActivitySummaryConfig, team: Team) -> bool:@@ -401,6 +420,7 @@ def feed_queue(config: ActivitySummaryConfig, now: datetime.datetime, team_ids:"""Move each source cursor forward and add the rows it passes to the queue state. Returns the pairs fed.A source with no cursor starts at the current end, so the backfill command seeds the earlier rows.+ `team_ids` defaults to the enrolled teams."""if team_ids is None:team_ids = enrolled_team_ids(config)@@ -600,7 +620,7 @@ def select_pairs(Stale pairs take at most `rebuild_budget_share` of `limit`, paying pairs first, so a version changedoes not stop the other updates. The first pass ranks the paying pairs and the free pairs with atleast `max_overdue_factor` staleness, for `paying_budget_share` of `limit`. The second pass ranks the- other free pairs for the budget that is left.+ other free pairs for the budget that is left. `team_ids` defaults to the enrolled teams."""if team_ids is None:team_ids = enrolled_team_ids(config)diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.pyindex 443d9376e104..40090309b69b 100644--- a/products/signals/backend/activity_summary/tick.py+++ b/products/signals/backend/activity_summary/tick.py@@ -41,7 +41,7 @@ from products.signals.backend.activity_summary.queue import (ActivitySummaryPair,ActivitySummaryTierStats,daily_usage,- enrolled_team_ids,+ enrollment,feed_queue,is_team_enrolled_for_generation,max_update_cost_usd,@@ -85,6 +85,13 @@ class ActivitySummaryTickInput:limit: int | None = None+@frozen+class EnrollmentCounts:+ qualifying_organizations: int = 0+ enrolled_organizations: int = 0+ enrolled_teams: int = 0++@frozenclass ActivitySummaryTickPlan:enabled: bool@@ -107,6 +114,7 @@ class ActivitySummaryTickPlan:max_error_rate: float = 1.0max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600tiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)+ enrollment: EnrollmentCounts = dataclasses.field(default_factory=EnrollmentCounts)@frozen@@ -130,6 +138,8 @@ class ActivitySummaryPairResult:status: strcost_usd: float | Nonecalled_model: bool+ # Has a default, so the result of a pair activity in flight during a deploy still decodes.+ new_summary: bool = False@frozen@@ -142,6 +152,10 @@ class ActivitySummaryTickResult:cost_usd: float = 0.0stop_reason: str | None = Nonetiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)+ enrollment: EnrollmentCounts = dataclasses.field(default_factory=EnrollmentCounts)+ # Successful updates that made a pair's first summary, and successful updates of an existing summary.+ new_builds: int = 0+ refresh_builds: int = 0def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPlan:@@ -151,16 +165,22 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP# Every pair would fail on the same missing gateway, so the tick runs none.logger.warning("activity_summary_tick_gateway_not_configured")enabled = config.enabled and not gateway_missing- # One enrollment read serves the feed and the selection.- team_ids = enrolled_team_ids(config)+ # One enrollment read serves the feed, the selection and the counts.+ enrolled = enrollment(config)+ counts = EnrollmentCounts(+ qualifying_organizations=len(enrolled.qualifying_organization_ids),+ enrolled_organizations=len(enrolled.organization_ids),+ enrolled_teams=len(enrolled.team_ids),+ )+ logger.info("activity_summary_enrollment_read", **dataclasses.asdict(counts))if config.enabled:# The cursors move even when a cap stops the tick, so new rows do not pile up behind them.- feed_queue(config, now, team_ids)+ feed_queue(config, now, enrolled.team_ids)usage = daily_usage(now) if enabled else Noneupdates_left = max(0, config.max_pairs_per_day - usage.updates) if usage else 0cost_left = max(0.0, config.max_cost_usd_per_day - usage.cost_usd) if usage else 0.0tick_limit = limit or config.max_pairs_per_tick- selection = select_pairs(config, now, tick_limit, team_ids) if updates_left and cost_left else None+ selection = select_pairs(config, now, tick_limit, enrolled.team_ids) if updates_left and cost_left else Nonepairs = selection.pairs if selection else []due_pairs = selection.due_pairs if selection else 0@@ -189,6 +209,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPmax_error_rate=config.max_error_rate,max_skip_backoff_hours=config.max_skip_backoff_hours,tiers=selection.tiers if selection else [],+ enrollment=counts,stop_reason=stop_reason,rebuild_before=config.rebuild_before,max_versions=config.max_versions,@@ -292,7 +313,12 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A_capture_generated(input.team_id, input.user_id, update)except Exception:logger.exception("activity_summary_capture_failed", team_id=input.team_id, user_id=input.user_id)- return ActivitySummaryPairResult(status=update.status.value, cost_usd=update.cost_usd, called_model=called_model)+ return ActivitySummaryPairResult(+ status=update.status.value,+ cost_usd=update.cost_usd,+ called_model=called_model,+ new_summary=update.new_summary,+ )def capture_tick(result: ActivitySummaryTickResult) -> None:@@ -305,6 +331,9 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:cost_usd=result.cost_usd,stop_reason=result.stop_reason,tiers=[dataclasses.asdict(tier) for tier in result.tiers],+ new_builds=result.new_builds,+ refresh_builds=result.refresh_builds,+ **dataclasses.asdict(result.enrollment),)ph_background_capture()(distinct_id=PIPELINE_DISTINCT_ID,@@ -317,6 +346,9 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:"errored": result.errored,"cost_usd": result.cost_usd,"stop_reason": result.stop_reason,+ "new_builds": result.new_builds,+ "refresh_builds": result.refresh_builds,+ **dataclasses.asdict(result.enrollment),**{f"{tier.tier}_{name}": getattr(tier, name)for tier in result.tiers@@ -363,6 +395,7 @@ class ActivitySummaryTickWorkflow:return ActivitySummaryTickResult(stop_reason=plan.stop_reason)statuses: Counter[str] = Counter()+ builds: Counter[bool] = Counter()errored = 0cost = 0.0# Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.@@ -422,6 +455,8 @@ class ActivitySummaryTickWorkflow:cost += plan.max_update_cost_usdcontinuestatuses[outcome.status] += 1+ if outcome.status == UpdateStatus.OK:+ builds[outcome.new_summary] += 1if outcome.called_model:calls += 1failures += outcome.status == UpdateStatus.FAILED@@ -437,6 +472,9 @@ class ActivitySummaryTickWorkflow:cost_usd=cost,stop_reason=stop_reason,tiers=plan.tiers,+ enrollment=plan.enrollment,+ new_builds=builds[True],+ refresh_builds=builds[False],)await workflow.execute_activity(capture_activity_summary_tick_activity,diff --git a/products/signals/backend/management/commands/preview_activity_summary_enrollment.py b/products/signals/backend/management/commands/preview_activity_summary_enrollment.pynew file mode 100644index 000000000000..940fc6d739d8--- /dev/null+++ b/products/signals/backend/management/commands/preview_activity_summary_enrollment.py@@ -0,0 +1,53 @@+"""Show who a self-driving percentage would enroll for activity summary generation. The command changes nothing.++It compares the current `activity-summary` config with the same config at the given percentage, so an+operator can check each rollout step before changing the flag payload.+"""++import json+import dataclasses++from django.core.management.base import BaseCommand, CommandError++from products.signals.backend.activity_summary.config import parse_config, read_config+from products.signals.backend.activity_summary.queue import Enrollment, enrollment+++class Command(BaseCommand):+ help = "Print the organizations and teams that a self-driving percentage enrolls. Writes nothing."++ def add_arguments(self, parser):+ parser.add_argument("--percentage", type=float, required=True, help="The percentage to preview, 0 to 100.")+ parser.add_argument(+ "--config",+ type=json.loads,+ default=None,+ help="A flag payload as JSON, in place of the `activity-summary` flag.",+ )+ parser.add_argument("--list", action="store_true", help="Print the ids of the organizations that change.")++ def handle(self, *args, **options):+ percentage = options["percentage"]+ if not 0 <= percentage <= 100:+ raise CommandError("--percentage must be from 0 to 100.")+ current = parse_config(options["config"]) if options["config"] is not None else read_config()+ # The preview enables the config, so it also works before the first rollout step.+ preview = dataclasses.replace(current, enabled=True, self_driving_organization_percentage=percentage)+ before = enrollment(current)+ after = enrollment(preview)+ self._write(f"current ({current.self_driving_organization_percentage}%)", before)+ self._write(f"preview ({percentage}%)", after)+ added = sorted(set(after.organization_ids) - set(before.organization_ids))+ removed = sorted(set(before.organization_ids) - set(after.organization_ids))+ self.stdout.write(f"organizations added: {len(added)}, removed: {len(removed)}")+ if options["list"]:+ for organization_id in added:+ self.stdout.write(f"+ {organization_id}")+ for organization_id in removed:+ self.stdout.write(f"- {organization_id}")++ def _write(self, label: str, result: Enrollment) -> None:+ self.stdout.write(+ f"{label}: {len(result.qualifying_organization_ids)} qualifying organizations, "+ f"{len(result.organization_ids)} enrolled organizations, {len(result.team_ids)} enrolled teams"+ )diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.pyindex f418fbe28bae..feb1a485d17a 100644--- a/products/signals/backend/test/test_activity_summary_tick.py+++ b/products/signals/backend/test/test_activity_summary_tick.py@@ -33,6 +33,7 @@ from products.signals.backend.activity_summary.queue import (DailyUsage,daily_usage,enrolled_team_ids,+ enrollment,feed_queue,is_team_enrolled_for_generation,is_team_enrolled_for_read,@@ -59,6 +60,7 @@ from products.signals.backend.activity_summary.tick import (ActivitySummaryTickPlan,ActivitySummaryTickResult,ActivitySummaryTickWorkflow,+ EnrollmentCounts,plan_tick,run_pair,)@@ -506,6 +508,27 @@ class TestActivitySummaryQueue(BaseTest):if config.self_driving_organization_percentage and not config.organization_ids:assert sourceless.id not in enrolled_team_ids(config)+ @parameterized.expand([("percentage_0", 0.0, 0), ("percentage_100", 100.0, 1)])+ def test_enrollment_counts_qualifying_organizations_apart_from_enrolled_ones(+ self, _name: str, percentage: float, enrolled: int+ ) -> None:+ self.organization.is_ai_data_processing_approved = True+ self.organization.save()+ SignalSourceConfig.objects.create(+ team=self.team, source_product="error_tracking", source_type="issue_created", enabled=True+ )+ Team.objects.create(organization=self.organization, name="No source")+ sourceless_organization = Organization.objects.create(name="No source", is_ai_data_processing_approved=True)+ Team.objects.create(organization=sourceless_organization, name="No source")++ result = enrollment(+ dataclasses.replace(self._config(), organization_ids=(), self_driving_organization_percentage=percentage)+ )++ assert result.qualifying_organization_ids == [self.organization.id]+ assert result.organization_ids == [self.organization.id][:enrolled]+ assert result.team_ids == [self.team.id][:enrolled]+def test_a_pair_is_not_updated_when_the_team_leaves_the_enrollment_before_dispatch(self) -> None:self._changes(self.user, 3)assert self._select() == [(self.team.id, self.user.id)]@@ -720,7 +743,7 @@ class TestActivitySummaryQueue(BaseTest):))- assert (result.status, result.called_model) == ("ok", True)+ assert (result.status, result.called_model, result.new_summary) == ("ok", True, True)assert emit.call_args.kwargs["document_id"] == f"{self.user.id}:2026-10-19"assert emit.call_args.kwargs["content"] == "Builds the signup funnel.\nFocus areas: signup funnel"event = capture.call_args.kwargs@@ -764,6 +787,7 @@ def _plan(pairs: int, **overrides: Any) -> ActivitySummaryTickPlan:"limits": DEFAULT_LIMITS,"max_error_rate": 0.5,"tiers": [ActivitySummaryTierStats(tier="paying", due_pairs=pairs)],+ "enrollment": EnrollmentCounts(qualifying_organizations=3, enrolled_organizations=2, enrolled_teams=4),"stop_reason": None,}return ActivitySummaryTickPlan(**{**fields, **overrides})@@ -786,7 +810,9 @@ async def _run_tick(peak = max(peak, running)await asyncio.sleep(0.01)running -= 1- return ActivitySummaryPairResult(status=pair_status, cost_usd=pair_cost, called_model=True)+ return ActivitySummaryPairResult(+ status=pair_status, cost_usd=pair_cost, called_model=True, new_summary=input.user_id % 2 == 0+ )@activity.defn(name="capture_activity_summary_tick_activity")async def fake_capture(result: ActivitySummaryTickResult) -> None:@@ -811,24 +837,31 @@ async def _run_tick(@pytest.mark.asyncio@pytest.mark.parametrize(- "plan,pair_cost,pair_status,picked,stop_reason",+ "plan,pair_cost,pair_status,picked,stop_reason,builds",[- (_plan(5), 0.01, "ok", 5, None),- (_plan(5, updates_left=3), 0.01, "ok", 3, STOP_MAX_PAIRS_PER_DAY),+ (_plan(5), 0.01, "ok", 5, None, (3, 2)),+ (_plan(5, updates_left=3), 0.01, "ok", 3, STOP_MAX_PAIRS_PER_DAY, (2, 1)),# A missing cost counts at the bound, so the second wave spends the last of the budget.- (_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY),+ (_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY, (2, 2)),# The breaker waits for a sample of 10 model calls, which is five waves.- (_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE),+ (_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE, (0, 0)),],)async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps_and_the_error_rate(- plan: ActivitySummaryTickPlan, pair_cost: float | None, pair_status: str, picked: int, stop_reason: str | None+ plan: ActivitySummaryTickPlan,+ pair_cost: float | None,+ pair_status: str,+ picked: int,+ stop_reason: str | None,+ builds: tuple[int, int],) -> None:result, peak, captured = await _run_tick(plan, pair_cost, pair_status)assert peak <= plan.max_concurrencyassert (result.picked, result.statuses, result.stop_reason) == (picked, {pair_status: picked}, stop_reason)assert result.tiers == plan.tiers+ assert (result.new_builds, result.refresh_builds) == builds+ assert result.enrollment == plan.enrollmentassert captured == [result]