Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): count activity summary enrollment and builds, add a preview (#122728)

posthog[bot]946dea8b8fb0
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index 4fe64bc5d5b3..7cd019195e57 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -79,6 +79,8 @@ class ActivitySummaryUpdate:
inputs_used: dict[str, Any] | None = None
input_watermark: datetime.datetime | None = None
rebuilt: bool = False
+ # The pair had no generated summary before this update.
+ new_summary: bool = False
class StoredSummaryVersion(Protocol):
@@ -317,6 +319,7 @@ def update_activity_summary(
inputs_used=inputs_used,
input_watermark=watermark if outcome.summary is not None else None,
rebuilt=rebuild,
+ new_summary=stored is None or stored.generated_at is None,
)
if dry_run:
return result
diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.py
index 5d11e5f4b7bd..0391e46aa011 100644
--- a/products/signals/backend/activity_summary/queue.py
+++ b/products/signals/backend/activity_summary/queue.py
@@ -177,32 +177,51 @@ def _in_rollout(config: ActivitySummaryConfig, organization_id: uuid.UUID) -> bo
return organization_rollout_bucket(organization_id) < threshold
-def enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:
+@frozen
+class Enrollment:
+ team_ids: list[int]
+ organization_ids: list[uuid.UUID]
+ # The organizations that a 100 percent rollout enrolls, with or without manual enrollment.
+ qualifying_organization_ids: list[uuid.UUID]
+
+
+def enrollment(config: ActivitySummaryConfig) -> Enrollment:
"""The teams that `is_team_enrolled_for_generation` enrolls, in a fixed number of queries."""
if not config.enabled:
- return []
+ return Enrollment(team_ids=[], organization_ids=[], qualifying_organization_ids=[])
+ from products.signals.backend.facade.api import (
+ team_ids_with_enabled_source, # noqa: PLC0415 — the facade imports this module through `read`
+ )
+
listed = _parse_organization_ids(config.organization_ids)
teams = (
Team.objects.filter(organization__is_ai_data_processing_approved=True)
.exclude(id__in=config.skip_team_ids)
.exclude(organization_id__in=_parse_organization_ids(config.deny_organization_ids))
)
+ # The teams with an enabled source come first, because they are far fewer than the approved teams.
+ with_source: dict[int, uuid.UUID] = dict(
+ teams.filter(id__in=team_ids_with_enabled_source()).values_list("id", "organization_id")
+ )
+ enrolled: dict[int, uuid.UUID]
if config.enroll_all_approved:
- return sorted(teams.values_list("id", flat=True))
- team_ids = set(teams.filter(organization_id__in=listed).values_list("id", flat=True)) if listed else set()
- if config.self_driving_organization_percentage > 0:
- from products.signals.backend.facade.api import (
- team_ids_with_enabled_source, # noqa: PLC0415 — the facade imports this module through `read`
- )
-
- # The teams with an enabled source come first, because they are far fewer than the approved teams.
- candidates = teams.filter(id__in=team_ids_with_enabled_source()).exclude(organization_id__in=listed)
- team_ids |= {
- team_id
- for team_id, organization_id in candidates.values_list("id", "organization_id")
+ enrolled = dict(teams.values_list("id", "organization_id"))
+ else:
+ enrolled = dict(teams.filter(organization_id__in=listed).values_list("id", "organization_id")) if listed else {}
+ enrolled |= {
+ team_id: organization_id
+ for team_id, organization_id in with_source.items()
if _in_rollout(config, organization_id)
}
- return sorted(team_ids)
+ return Enrollment(
+ team_ids=sorted(enrolled),
+ organization_ids=sorted(set(enrolled.values())),
+ qualifying_organization_ids=sorted(set(with_source.values())),
+ )
+
+
+def enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:
+ return enrollment(config).team_ids
def is_team_enrolled_for_generation(config: ActivitySummaryConfig, team: Team) -> bool:
@@ -401,6 +420,7 @@ def feed_queue(config: ActivitySummaryConfig, now: datetime.datetime, team_ids:
"""Move each source cursor forward and add the rows it passes to the queue state. Returns the pairs fed.
A source with no cursor starts at the current end, so the backfill command seeds the earlier rows.
+ `team_ids` defaults to the enrolled teams.
"""
if team_ids is None:
team_ids = enrolled_team_ids(config)
@@ -600,7 +620,7 @@ def select_pairs(
Stale pairs take at most `rebuild_budget_share` of `limit`, paying pairs first, so a version change
does not stop the other updates. The first pass ranks the paying pairs and the free pairs with at
least `max_overdue_factor` staleness, for `paying_budget_share` of `limit`. The second pass ranks the
- other free pairs for the budget that is left.
+ other free pairs for the budget that is left. `team_ids` defaults to the enrolled teams.
"""
if team_ids is None:
team_ids = enrolled_team_ids(config)
diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.py
index 443d9376e104..40090309b69b 100644
--- a/products/signals/backend/activity_summary/tick.py
+++ b/products/signals/backend/activity_summary/tick.py
@@ -41,7 +41,7 @@ from products.signals.backend.activity_summary.queue import (
ActivitySummaryPair,
ActivitySummaryTierStats,
daily_usage,
- enrolled_team_ids,
+ enrollment,
feed_queue,
is_team_enrolled_for_generation,
max_update_cost_usd,
@@ -85,6 +85,13 @@ class ActivitySummaryTickInput:
limit: int | None = None
+@frozen
+class EnrollmentCounts:
+ qualifying_organizations: int = 0
+ enrolled_organizations: int = 0
+ enrolled_teams: int = 0
+
+
@frozen
class ActivitySummaryTickPlan:
enabled: bool
@@ -107,6 +114,7 @@ class ActivitySummaryTickPlan:
max_error_rate: float = 1.0
max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600
tiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)
+ enrollment: EnrollmentCounts = dataclasses.field(default_factory=EnrollmentCounts)
@frozen
@@ -130,6 +138,8 @@ class ActivitySummaryPairResult:
status: str
cost_usd: float | None
called_model: bool
+ # Has a default, so the result of a pair activity in flight during a deploy still decodes.
+ new_summary: bool = False
@frozen
@@ -142,6 +152,10 @@ class ActivitySummaryTickResult:
cost_usd: float = 0.0
stop_reason: str | None = None
tiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)
+ enrollment: EnrollmentCounts = dataclasses.field(default_factory=EnrollmentCounts)
+ # Successful updates that made a pair's first summary, and successful updates of an existing summary.
+ new_builds: int = 0
+ refresh_builds: int = 0
def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPlan:
@@ -151,16 +165,22 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
# Every pair would fail on the same missing gateway, so the tick runs none.
logger.warning("activity_summary_tick_gateway_not_configured")
enabled = config.enabled and not gateway_missing
- # One enrollment read serves the feed and the selection.
- team_ids = enrolled_team_ids(config)
+ # One enrollment read serves the feed, the selection and the counts.
+ enrolled = enrollment(config)
+ counts = EnrollmentCounts(
+ qualifying_organizations=len(enrolled.qualifying_organization_ids),
+ enrolled_organizations=len(enrolled.organization_ids),
+ enrolled_teams=len(enrolled.team_ids),
+ )
+ logger.info("activity_summary_enrollment_read", **dataclasses.asdict(counts))
if config.enabled:
# The cursors move even when a cap stops the tick, so new rows do not pile up behind them.
- feed_queue(config, now, team_ids)
+ feed_queue(config, now, enrolled.team_ids)
usage = daily_usage(now) if enabled else None
updates_left = max(0, config.max_pairs_per_day - usage.updates) if usage else 0
cost_left = max(0.0, config.max_cost_usd_per_day - usage.cost_usd) if usage else 0.0
tick_limit = limit or config.max_pairs_per_tick
- selection = select_pairs(config, now, tick_limit, team_ids) if updates_left and cost_left else None
+ selection = select_pairs(config, now, tick_limit, enrolled.team_ids) if updates_left and cost_left else None
pairs = selection.pairs if selection else []
due_pairs = selection.due_pairs if selection else 0
@@ -189,6 +209,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
max_error_rate=config.max_error_rate,
max_skip_backoff_hours=config.max_skip_backoff_hours,
tiers=selection.tiers if selection else [],
+ enrollment=counts,
stop_reason=stop_reason,
rebuild_before=config.rebuild_before,
max_versions=config.max_versions,
@@ -292,7 +313,12 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A
_capture_generated(input.team_id, input.user_id, update)
except Exception:
logger.exception("activity_summary_capture_failed", team_id=input.team_id, user_id=input.user_id)
- return ActivitySummaryPairResult(status=update.status.value, cost_usd=update.cost_usd, called_model=called_model)
+ return ActivitySummaryPairResult(
+ status=update.status.value,
+ cost_usd=update.cost_usd,
+ called_model=called_model,
+ new_summary=update.new_summary,
+ )
def capture_tick(result: ActivitySummaryTickResult) -> None:
@@ -305,6 +331,9 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:
cost_usd=result.cost_usd,
stop_reason=result.stop_reason,
tiers=[dataclasses.asdict(tier) for tier in result.tiers],
+ new_builds=result.new_builds,
+ refresh_builds=result.refresh_builds,
+ **dataclasses.asdict(result.enrollment),
)
ph_background_capture()(
distinct_id=PIPELINE_DISTINCT_ID,
@@ -317,6 +346,9 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:
"errored": result.errored,
"cost_usd": result.cost_usd,
"stop_reason": result.stop_reason,
+ "new_builds": result.new_builds,
+ "refresh_builds": result.refresh_builds,
+ **dataclasses.asdict(result.enrollment),
**{
f"{tier.tier}_{name}": getattr(tier, name)
for tier in result.tiers
@@ -363,6 +395,7 @@ class ActivitySummaryTickWorkflow:
return ActivitySummaryTickResult(stop_reason=plan.stop_reason)
statuses: Counter[str] = Counter()
+ builds: Counter[bool] = Counter()
errored = 0
cost = 0.0
# Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.
@@ -422,6 +455,8 @@ class ActivitySummaryTickWorkflow:
cost += plan.max_update_cost_usd
continue
statuses[outcome.status] += 1
+ if outcome.status == UpdateStatus.OK:
+ builds[outcome.new_summary] += 1
if outcome.called_model:
calls += 1
failures += outcome.status == UpdateStatus.FAILED
@@ -437,6 +472,9 @@ class ActivitySummaryTickWorkflow:
cost_usd=cost,
stop_reason=stop_reason,
tiers=plan.tiers,
+ enrollment=plan.enrollment,
+ new_builds=builds[True],
+ refresh_builds=builds[False],
)
await workflow.execute_activity(
capture_activity_summary_tick_activity,
diff --git a/products/signals/backend/management/commands/preview_activity_summary_enrollment.py b/products/signals/backend/management/commands/preview_activity_summary_enrollment.py
new file mode 100644
index 000000000000..940fc6d739d8
--- /dev/null
+++ b/products/signals/backend/management/commands/preview_activity_summary_enrollment.py
@@ -0,0 +1,53 @@
+"""Show who a self-driving percentage would enroll for activity summary generation. The command changes nothing.
+
+It compares the current `activity-summary` config with the same config at the given percentage, so an
+operator can check each rollout step before changing the flag payload.
+"""
+
+import json
+import dataclasses
+
+from django.core.management.base import BaseCommand, CommandError
+
+from products.signals.backend.activity_summary.config import parse_config, read_config
+from products.signals.backend.activity_summary.queue import Enrollment, enrollment
+
+
+class Command(BaseCommand):
+ help = "Print the organizations and teams that a self-driving percentage enrolls. Writes nothing."
+
+ def add_arguments(self, parser):
+ parser.add_argument("--percentage", type=float, required=True, help="The percentage to preview, 0 to 100.")
+ parser.add_argument(
+ "--config",
+ type=json.loads,
+ default=None,
+ help="A flag payload as JSON, in place of the `activity-summary` flag.",
+ )
+ parser.add_argument("--list", action="store_true", help="Print the ids of the organizations that change.")
+
+ def handle(self, *args, **options):
+ percentage = options["percentage"]
+ if not 0 <= percentage <= 100:
+ raise CommandError("--percentage must be from 0 to 100.")
+ current = parse_config(options["config"]) if options["config"] is not None else read_config()
+ # The preview enables the config, so it also works before the first rollout step.
+ preview = dataclasses.replace(current, enabled=True, self_driving_organization_percentage=percentage)
+ before = enrollment(current)
+ after = enrollment(preview)
+ self._write(f"current ({current.self_driving_organization_percentage}%)", before)
+ self._write(f"preview ({percentage}%)", after)
+ added = sorted(set(after.organization_ids) - set(before.organization_ids))
+ removed = sorted(set(before.organization_ids) - set(after.organization_ids))
+ self.stdout.write(f"organizations added: {len(added)}, removed: {len(removed)}")
+ if options["list"]:
+ for organization_id in added:
+ self.stdout.write(f"+ {organization_id}")
+ for organization_id in removed:
+ self.stdout.write(f"- {organization_id}")
+
+ def _write(self, label: str, result: Enrollment) -> None:
+ self.stdout.write(
+ f"{label}: {len(result.qualifying_organization_ids)} qualifying organizations, "
+ f"{len(result.organization_ids)} enrolled organizations, {len(result.team_ids)} enrolled teams"
+ )
diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.py
index f418fbe28bae..feb1a485d17a 100644
--- a/products/signals/backend/test/test_activity_summary_tick.py
+++ b/products/signals/backend/test/test_activity_summary_tick.py
@@ -33,6 +33,7 @@ from products.signals.backend.activity_summary.queue import (
DailyUsage,
daily_usage,
enrolled_team_ids,
+ enrollment,
feed_queue,
is_team_enrolled_for_generation,
is_team_enrolled_for_read,
@@ -59,6 +60,7 @@ from products.signals.backend.activity_summary.tick import (
ActivitySummaryTickPlan,
ActivitySummaryTickResult,
ActivitySummaryTickWorkflow,
+ EnrollmentCounts,
plan_tick,
run_pair,
)
@@ -506,6 +508,27 @@ class TestActivitySummaryQueue(BaseTest):
if config.self_driving_organization_percentage and not config.organization_ids:
assert sourceless.id not in enrolled_team_ids(config)
+ @parameterized.expand([("percentage_0", 0.0, 0), ("percentage_100", 100.0, 1)])
+ def test_enrollment_counts_qualifying_organizations_apart_from_enrolled_ones(
+ self, _name: str, percentage: float, enrolled: int
+ ) -> None:
+ self.organization.is_ai_data_processing_approved = True
+ self.organization.save()
+ SignalSourceConfig.objects.create(
+ team=self.team, source_product="error_tracking", source_type="issue_created", enabled=True
+ )
+ Team.objects.create(organization=self.organization, name="No source")
+ sourceless_organization = Organization.objects.create(name="No source", is_ai_data_processing_approved=True)
+ Team.objects.create(organization=sourceless_organization, name="No source")
+
+ result = enrollment(
+ dataclasses.replace(self._config(), organization_ids=(), self_driving_organization_percentage=percentage)
+ )
+
+ assert result.qualifying_organization_ids == [self.organization.id]
+ assert result.organization_ids == [self.organization.id][:enrolled]
+ assert result.team_ids == [self.team.id][:enrolled]
+
def test_a_pair_is_not_updated_when_the_team_leaves_the_enrollment_before_dispatch(self) -> None:
self._changes(self.user, 3)
assert self._select() == [(self.team.id, self.user.id)]
@@ -720,7 +743,7 @@ class TestActivitySummaryQueue(BaseTest):
)
)
- assert (result.status, result.called_model) == ("ok", True)
+ assert (result.status, result.called_model, result.new_summary) == ("ok", True, True)
assert emit.call_args.kwargs["document_id"] == f"{self.user.id}:2026-10-19"
assert emit.call_args.kwargs["content"] == "Builds the signup funnel.\nFocus areas: signup funnel"
event = capture.call_args.kwargs
@@ -764,6 +787,7 @@ def _plan(pairs: int, **overrides: Any) -> ActivitySummaryTickPlan:
"limits": DEFAULT_LIMITS,
"max_error_rate": 0.5,
"tiers": [ActivitySummaryTierStats(tier="paying", due_pairs=pairs)],
+ "enrollment": EnrollmentCounts(qualifying_organizations=3, enrolled_organizations=2, enrolled_teams=4),
"stop_reason": None,
}
return ActivitySummaryTickPlan(**{**fields, **overrides})
@@ -786,7 +810,9 @@ async def _run_tick(
peak = max(peak, running)
await asyncio.sleep(0.01)
running -= 1
- return ActivitySummaryPairResult(status=pair_status, cost_usd=pair_cost, called_model=True)
+ return ActivitySummaryPairResult(
+ status=pair_status, cost_usd=pair_cost, called_model=True, new_summary=input.user_id % 2 == 0
+ )
@activity.defn(name="capture_activity_summary_tick_activity")
async def fake_capture(result: ActivitySummaryTickResult) -> None:
@@ -811,24 +837,31 @@ async def _run_tick(
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "plan,pair_cost,pair_status,picked,stop_reason",
+ "plan,pair_cost,pair_status,picked,stop_reason,builds",
[
- (_plan(5), 0.01, "ok", 5, None),
- (_plan(5, updates_left=3), 0.01, "ok", 3, STOP_MAX_PAIRS_PER_DAY),
+ (_plan(5), 0.01, "ok", 5, None, (3, 2)),
+ (_plan(5, updates_left=3), 0.01, "ok", 3, STOP_MAX_PAIRS_PER_DAY, (2, 1)),
# A missing cost counts at the bound, so the second wave spends the last of the budget.
- (_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY),
+ (_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY, (2, 2)),
# The breaker waits for a sample of 10 model calls, which is five waves.
- (_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE),
+ (_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE, (0, 0)),
],
)
async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps_and_the_error_rate(
- plan: ActivitySummaryTickPlan, pair_cost: float | None, pair_status: str, picked: int, stop_reason: str | None
+ plan: ActivitySummaryTickPlan,
+ pair_cost: float | None,
+ pair_status: str,
+ picked: int,
+ stop_reason: str | None,
+ builds: tuple[int, int],
) -> None:
result, peak, captured = await _run_tick(plan, pair_cost, pair_status)
assert peak <= plan.max_concurrency
assert (result.picked, result.statuses, result.stop_reason) == (picked, {pair_status: picked}, stop_reason)
assert result.tiers == plan.tiers
+ assert (result.new_builds, result.refresh_builds) == builds
+ assert result.enrollment == plan.enrollment
assert captured == [result]