:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): prioritize activity summary pairs by tier, activity and staleness (#121717)
posthog[bot]c16f9819811e
diff --git a/products/signals/backend/activity_summary/config.py b/products/signals/backend/activity_summary/config.pyindex bf3b7a301405..b4f710115980 100644--- a/products/signals/backend/activity_summary/config.py+++ b/products/signals/backend/activity_summary/config.py@@ -14,7 +14,12 @@ import structlogfrom posthog.dataclasses import frozen-from products.signals.backend.activity_summary.logic import MAX_VERSIONS, MIN_UNITS, MIN_UPDATE_INTERVAL+from products.signals.backend.activity_summary.logic import (+ MAX_SKIP_BACKOFF,+ MAX_VERSIONS,+ MIN_UNITS,+ MIN_UPDATE_INTERVAL,+)from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, RenderLimitsfrom products.signals.backend.scout_harness.team_limits import read_flag_payload@@ -44,6 +49,20 @@ class ActivitySummaryConfig:capture_events: bool = True# Count viewed objects in the queue. Turn it on only when the update reads views as an input.count_views: bool = False+ # The summary age each tier aims for. A pair's staleness is its summary age over this interval.+ target_interval_days_paying: float = 3.0+ target_interval_days_free: float = 5.0+ # A free pair with at least this staleness joins the paying pass, so paying pairs cannot starve it.+ max_overdue_factor: float = 3.0+ # Added to the score of a pair with no summary, because a first summary changes the most.+ cold_start_bonus: float = 5.0+ # The share of the tick that the paying pass can take. The free pass gets the rest.+ paying_budget_share: float = 1.0+ # The largest share of one pass that the pairs of one organization can take.+ max_org_share: float = 0.1+ # The tick stops when this share of its model calls failed or raised.+ max_error_rate: float = 0.5+ max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600limits: RenderLimits = DEFAULT_LIMITS# Summaries generated before this time are rebuilt from the full window, as after a version change.rebuild_before: datetime.datetime | None = None@@ -132,6 +151,18 @@ def parse_config(payload: Any) -> ActivitySummaryConfig:emit_embeddings=_bool(payload.get("emit_embeddings"), defaults.emit_embeddings),capture_events=_bool(payload.get("capture_events"), defaults.capture_events),count_views=_bool(payload.get("count_views"), defaults.count_views),+ target_interval_days_paying=_positive_number(+ payload.get("target_interval_days_paying"), defaults.target_interval_days_paying+ ),+ target_interval_days_free=_positive_number(+ payload.get("target_interval_days_free"), defaults.target_interval_days_free+ ),+ max_overdue_factor=_positive_number(payload.get("max_overdue_factor"), defaults.max_overdue_factor),+ cold_start_bonus=_positive_number(payload.get("cold_start_bonus"), defaults.cold_start_bonus),+ paying_budget_share=_fraction(payload.get("paying_budget_share"), defaults.paying_budget_share),+ max_org_share=_fraction(payload.get("max_org_share"), defaults.max_org_share),+ max_error_rate=_fraction(payload.get("max_error_rate"), defaults.max_error_rate),+ max_skip_backoff_hours=_positive_number(payload.get("max_skip_backoff_hours"), defaults.max_skip_backoff_hours),limits=parse_limits(payload.get("limits")),rebuild_before=_timestamp(payload.get("rebuild_before")),rebuild_budget_share=_fraction(payload.get("rebuild_budget_share"), defaults.rebuild_budget_share),diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.pyindex 1dbc3bb8a868..3a6ed5d72db8 100644--- a/products/signals/backend/activity_summary/logic.py+++ b/products/signals/backend/activity_summary/logic.py@@ -37,7 +37,11 @@ MIN_UPDATE_INTERVAL = datetime.timedelta(days=1)# The `UserActivitySummaryVersion` rows kept per summary. An update deletes the older ones.MAX_VERSIONS = 30# A pair that skipped or failed waits this long, so it does not take the queue head on every tick.+# Each further skip since the last good update doubles the wait, up to the `max_skip_backoff` of the update.RETRY_BACKOFF = datetime.timedelta(hours=6)+MAX_SKIP_BACKOFF = datetime.timedelta(hours=72)+# Enough doublings to pass any sane maximum, and a small enough exponent for `timedelta`.+_MAX_SKIP_DOUBLINGS = 16# The error prefixes of an update that raised before it stored a result, before and after the model call.EXCEPTION_ERROR_PREFIX = "exception: "EXCEPTION_AFTER_MODEL_CALL_ERROR_PREFIX = "exception_after_model_call: "@@ -99,22 +103,39 @@ def is_stale(stored: StoredSummaryVersion | None, rebuild_before: datetime.datetreturn rebuild_before is not None and stored.generated_at < rebuild_before+def skip_backoff(consecutive_skips: int, max_skip_backoff: datetime.timedelta) -> datetime.timedelta:+ """The wait after a skip for too few units, when `consecutive_skips` skips came before it."""+ return max(RETRY_BACKOFF, min(RETRY_BACKOFF * 2 ** min(consecutive_skips, _MAX_SKIP_DOUBLINGS), max_skip_backoff))++def _input_start(watermark: datetime.datetime | None, now: datetime.datetime, window_days: int) -> datetime.datetime:earliest = now - datetime.timedelta(days=window_days)return earliest if watermark is None else max(earliest, watermark + _WATERMARK_STEP)def _back_off(- team_id: int, user_id: int, stored: UserActivitySummary | None, next_eligible_at: datetime.datetime+ team_id: int,+ user_id: int,+ stored: UserActivitySummary | None,+ now: datetime.datetime,+ max_skip_backoff: datetime.timedelta,) -> None:+ skips = stored.consecutive_skips if stored is not None else 0+ next_eligible_at = now + skip_backoff(skips, max_skip_backoff)# `update()` leaves `updated_at` alone, so the daily caps do not count a skip as a model call.if stored is not None:- UserActivitySummary.objects.for_team(team_id).filter(id=stored.id).update(next_eligible_at=next_eligible_at)+ UserActivitySummary.objects.for_team(team_id).filter(id=stored.id).update(+ next_eligible_at=next_eligible_at, consecutive_skips=min(skips + 1, _MAX_SKIP_DOUBLINGS)+ )returnUserActivitySummary.objects.for_team(team_id).get_or_create(team_id=team_id,user_id=user_id,- defaults={"status": UserActivitySummary.Status.SKIPPED, "next_eligible_at": next_eligible_at},+ defaults={+ "status": UserActivitySummary.Status.SKIPPED,+ "next_eligible_at": next_eligible_at,+ "consecutive_skips": 1,+ },)@@ -218,6 +239,7 @@ def update_activity_summary(limits: RenderLimits = DEFAULT_LIMITS,rebuild_before: datetime.datetime | None = None,max_versions: int = MAX_VERSIONS,+ max_skip_backoff: datetime.timedelta = MAX_SKIP_BACKOFF,) -> ActivitySummaryUpdate:"""Update the stored activity summary of one user in one project.@@ -225,7 +247,8 @@ def update_activity_summary(run or for a stale summary (see `is_stale`). A stale summary is not sent to the model. A failedupdate records the error and keeps the stored summary and the watermark. With `dry_run`, themodel is still called but nothing is written. `force` ignores `next_eligible_at`.- A skip for too few units and a failed update set `next_eligible_at` to `RETRY_BACKOFF` from now.+ A failed update sets `next_eligible_at` to `RETRY_BACKOFF` from now. A skip for too few units sets it+ to `skip_backoff` from now, which doubles with each skip until a good update resets it.A successful update also appends a `UserActivitySummaryVersion` and keeps the latest `max_versions`."""team = Team.objects.select_related("organization").get(id=team_id)@@ -253,7 +276,7 @@ def update_activity_summary()if rendered is None or rendered.units < min_units:if not dry_run:- _back_off(team_id, user_id, stored, now + RETRY_BACKOFF)+ _back_off(team_id, user_id, stored, now, max_skip_backoff)return ActivitySummaryUpdate(status=UpdateStatus.SKIPPED_TOO_FEW_UNITS,rendered_input=rendered.text if rendered else None,@@ -332,6 +355,7 @@ def update_activity_summary("render_version": RENDER_VERSION,"input_watermark": watermark,"next_eligible_at": now + min_update_interval,+ "consecutive_skips": 0,},max_versions,)diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.pyindex a1d2752e9bd5..1ed5def686db 100644--- a/products/signals/backend/activity_summary/queue.py+++ b/products/signals/backend/activity_summary/queue.py@@ -1,9 +1,10 @@"""Pick the (team, user) pairs that the next activity summary tick updates, and measure the daily caps.Each pair has a queue state row. One cursor per input source reads the new rows of all enrolled teams-and adds them to the pending units of their pair. `select_pairs` then reads the due pairs in one-ordered query. Row counts here are only a queue estimate. `update_activity_summary` still applies the-real drop rules and `min_units`, and backs a pair off when it skips, so the pair leaves the queue head.+and adds them to the pending units of their pair. `select_pairs` then ranks the due pairs and fills the+tick budget in two passes: paying organizations first, then free organizations with the budget that is+left. Row counts here are only a queue estimate. `update_activity_summary` still applies the real drop+rules and `min_units`, and backs a pair off when it skips, so the pair leaves the queue head."""import math@@ -12,8 +13,25 @@ from collections.abc import Callablefrom typing import Anyfrom django.db import transaction-from django.db.models import BooleanField, Case, Count, ExpressionWrapper, F, Max, OuterRef, Q, Subquery, Sum, When-from django.db.models.functions import Coalesce, Greatest+from django.db.models import (+ BooleanField,+ Case,+ Count,+ DateTimeField,+ F,+ FloatField,+ Max,+ OuterRef,+ Q,+ QuerySet,+ Subquery,+ Sum,+ Value,+ When,+ Window,+)+from django.db.models.expressions import Combinable+from django.db.models.functions import Cast, Coalesce, Extract, Greatest, Ln, RowNumberfrom posthog.dataclasses import frozenfrom posthog.models import ActivityLog, FileSystemViewLog, Team@@ -54,14 +72,10 @@ SOURCE_VIEWS = "views"FEED_LAG = datetime.timedelta(minutes=5)# One tick reads at most this much of one source. A cursor that fell behind catches up over the next ticks.MAX_FEED_STEP = datetime.timedelta(hours=6)-# Pairs with no summary first, then the most pending units, then the oldest summary. The state index has this order.-_QUEUE_ORDER = (- ExpressionWrapper(Q(summarized_at__isnull=False), output_field=BooleanField()).asc(),- F("pending_units").desc(),- F("summarized_at").asc(),- "team_id",- "user_id",-)+TIER_PAYING = "paying"+TIER_FREE = "free"+# A summary younger than this many target intervals counts as fresh on the tick event.+_FRESH_STALENESS = 2@frozen(order=True)@@ -70,6 +84,23 @@ class ActivitySummaryPair:user_id: int+@frozen+class ActivitySummaryTierStats:+ tier: str+ due_pairs: int = 0+ # Hours since the due time of the pair that waits longest. Null when no pair is due.+ oldest_due_hours: float | None = None+ # The share of pairs with activity in the window whose summary is younger than twice the tier's target interval.+ fresh_share: float | None = None+++@frozen+class PairSelection:+ pairs: list[ActivitySummaryPair]+ due_pairs: int+ tiers: list[ActivitySummaryTierStats]++@frozenclass _NewRows:units: int@@ -372,41 +403,139 @@ def _current(config: ActivitySummaryConfig) -> Q:return Q(summarized_at__isnull=True) | current-def select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: int) -> list[ActivitySummaryPair]:+def _hours_since(now: datetime.datetime, at: Coalesce) -> Combinable:+ age = Value(now, output_field=DateTimeField()) - at+ return Cast(Extract(age, "epoch"), FloatField()) / Value(3600.0)+++def _states(config: ActivitySummaryConfig, now: datetime.datetime, team_ids: list[int]) -> QuerySet:+ """The queue states of these teams, with their tier, staleness and priority score.++ Staleness is the summary age over the tier's target interval. A pair with no summary counts its age+ from its first queued row. The score is log(1 + pending units) times staleness, plus a bonus for a+ pair with no summary, or a bonus plus staleness for a stale summary.+ """+ paying = Coalesce(F("team__organization__has_active_subscription"), Value(False))+ target_hours = Case(+ When(paying=True, then=Value(config.target_interval_days_paying * 24)),+ default=Value(config.target_interval_days_free * 24),+ output_field=FloatField(),+ )+ age_hours = _hours_since(now, Coalesce("summarized_at", "created_at"))+ stale = Case(When(_current(config), then=Value(False)), default=Value(True), output_field=BooleanField())+ cold_bonus = Case(+ When(summarized_at__isnull=True, then=Value(config.cold_start_bonus)),+ # A stale pair often has no pending units, so its staleness orders it among the other rebuilds.+ When(stale=True, then=Value(config.cold_start_bonus) + F("staleness")),+ default=Value(0.0),+ output_field=FloatField(),+ )+ return (+ # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)+ UserActivitySummaryQueueState.all_teams.filter(team_id__in=team_ids)+ .annotate(organization_id=F("team__organization_id"), paying=paying, stale=stale)+ .annotate(staleness=age_hours / target_hours)+ .annotate(score=Ln(F("pending_units") + 1) * F("staleness") + cold_bonus)+ )+++def _due(states: QuerySet, config: ActivitySummaryConfig, now: datetime.datetime) -> QuerySet:+ return states.filter(+ Q(due_at__isnull=True) | Q(due_at__lte=now),+ Q(pending_units__gte=max(1, config.min_units)) | Q(stale=True),+ last_activity_at__gte=now - datetime.timedelta(days=config.limits.window_days),+ )+++def _cap_rebuilds(due: QuerySet, max_rebuilds: int) -> QuerySet:+ """The due pairs with at most `max_rebuilds` stale pairs, paying pairs and the highest scores first."""+ rebuild_ids = list(+ due.filter(stale=True)+ .order_by("-paying", F("score").desc(), "team_id", "user_id")+ .values_list("id", flat=True)[:max_rebuilds]+ )+ return due.filter(Q(stale=False) | Q(id__in=rebuild_ids))+++def _fill_pass(due: QuerySet, limit: int, max_org_share: float) -> list[UserActivitySummaryQueueState]:+ """The highest scores first, with at most `max_org_share` of `limit` from one organization."""+ if limit <= 0:+ return []+ order = (F("score").desc(), "team_id", "user_id")+ return list(+ due.annotate(org_rank=Window(RowNumber(), partition_by=[F("organization_id")], order_by=order))+ .filter(org_rank__lte=max(1, int(max_org_share * limit)))+ .order_by(*order)+ .only("id", "team_id", "user_id")[:limit]+ )+++def _tier_stats(+ states: QuerySet, due: QuerySet, config: ActivitySummaryConfig, now: datetime.datetime+) -> list[ActivitySummaryTierStats]:+ window_start = now - datetime.timedelta(days=config.limits.window_days)+ waiting = {+ row["paying"]: row+ for row in due.values("paying").annotate(+ pairs=Count("id"),+ oldest=Max(_hours_since(now, Coalesce("due_at", "created_at"))),+ )+ }+ activity = {+ row["paying"]: row+ for row in states.filter(last_activity_at__gte=window_start)+ .values("paying")+ .annotate(+ active=Count("id"),+ fresh=Count("id", filter=Q(summarized_at__isnull=False, staleness__lt=_FRESH_STALENESS)),+ )+ }+ stats = []+ for tier, paying in ((TIER_PAYING, True), (TIER_FREE, False)):+ due_row = waiting.get(paying)+ activity_row = activity.get(paying)+ stats.append(+ ActivitySummaryTierStats(+ tier=tier,+ due_pairs=due_row["pairs"] if due_row else 0,+ oldest_due_hours=max(0.0, due_row["oldest"]) if due_row else None,+ fresh_share=activity_row["fresh"] / activity_row["active"] if activity_row else None,+ )+ )+ return stats+++def select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: int) -> PairSelection:"""The due pairs of the enrolled teams, highest priority first, cut to `limit`.A pair is due when it has a row in the last `window_days` and its due time is empty or past. A pairwith a current summary also needs at least `min_units` pending units. One unit never comes from lessthan one row, so a pair with fewer rows would only skip. A pair with a stale summary needs no pending- units, because its update reads the full window. Stale pairs take at most `rebuild_budget_share` of- `limit`, so a version change does not stop the other updates.-- Pairs with no summary come first, then stale pairs with the oldest summary first, then the pairs with- the most pending units, then the oldest summaries.+ units, because its update reads the full window, and it ranks with the pairs with no summary.+ Stale pairs take at most `rebuild_budget_share` of `limit`, paying pairs first, so a version change+ does not stop the other updates. The first pass ranks the paying pairs and the free pairs with at+ least `max_overdue_factor` staleness, for `paying_budget_share` of `limit`. The second pass ranks the+ other free pairs for the budget that is left."""team_ids = enrolled_team_ids(config)- if not team_ids or limit <= 0:- return []- window_start = now - datetime.timedelta(days=config.limits.window_days)- due = (- # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)- UserActivitySummaryQueueState.all_teams.filter(team_id__in=team_ids, last_activity_at__gte=window_start).filter(- Q(due_at__isnull=True) | Q(due_at__lte=now)- )+ if not team_ids:+ return PairSelection(pairs=[], due_pairs=0, tiers=[])+ states = _states(config, now, team_ids)+ due = _due(states, config, now)+ tiers = _tier_stats(states, due, config, now)+ due = _cap_rebuilds(due, math.ceil(limit * config.rebuild_budget_share))+ first = _fill_pass(+ due.filter(Q(paying=True) | Q(staleness__gte=config.max_overdue_factor)),+ int(limit * config.paying_budget_share),+ config.max_org_share,)- current = _current(config)- updates = list(- due.filter(current, pending_units__gte=max(1, config.min_units))- .order_by(*_QUEUE_ORDER)- .values_list("team_id", "user_id", "summarized_at")[:limit]+ second = _fill_pass(+ due.filter(paying=False).exclude(id__in=[state.id for state in first]),+ limit - len(first),+ config.max_org_share,)- rebuilds = list(- due.exclude(current)- .order_by("summarized_at", "team_id", "user_id")- .values_list("team_id", "user_id")[: math.ceil(limit * config.rebuild_budget_share)]+ return PairSelection(+ pairs=[ActivitySummaryPair(team_id=state.team_id, user_id=state.user_id) for state in first + second],+ due_pairs=sum(tier.due_pairs for tier in tiers),+ tiers=tiers,)- cold = [(team_id, user_id) for team_id, user_id, at in updates if at is None]- warm = [(team_id, user_id) for team_id, user_id, at in updates if at is not None]- return [ActivitySummaryPair(team_id=team_id, user_id=user_id) for team_id, user_id in cold + rebuilds + warm][- :limit- ]diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.pyindex 4a535621749e..b3ea7973a074 100644--- a/products/signals/backend/activity_summary/tick.py+++ b/products/signals/backend/activity_summary/tick.py@@ -6,6 +6,7 @@ tick off. The pair activity calls `update_activity_summary`, then queues the embimport asyncioimport datetime+import dataclassesfrom collections import Counterfrom django.conf import settings@@ -27,6 +28,7 @@ from posthog.temporal.common.utils import close_db_connectionsfrom products.signals.backend.activity_summary.config import read_configfrom products.signals.backend.activity_summary.logic import (+ MAX_SKIP_BACKOFF,MAX_VERSIONS,ActivitySummaryUpdate,UpdateAfterModelCallError,@@ -37,6 +39,7 @@ from products.signals.backend.activity_summary.logic import (from products.signals.backend.activity_summary.prompt import PROMPT_VERSIONfrom products.signals.backend.activity_summary.queue import (ActivitySummaryPair,+ ActivitySummaryTierStats,daily_usage,feed_queue,max_update_cost_usd,@@ -59,6 +62,12 @@ STOP_MAX_PAIRS_PER_TICK = "max_pairs_per_tick"STOP_MAX_PAIRS_PER_DAY = "max_pairs_per_day"STOP_MAX_COST_USD_PER_DAY = "max_cost_usd_per_day"STOP_GATEWAY_NOT_CONFIGURED = "gateway_not_configured"+# Due pairs are left, but each organization with due pairs already has its share of the tick.+STOP_MAX_ORG_SHARE = "max_org_share"+STOP_ERROR_RATE = "error_rate"++# The circuit breaker waits for this many model calls, so a few early failures do not stop the tick.+_MIN_ERROR_RATE_SAMPLE = 10_PLAN_TIMEOUT = datetime.timedelta(minutes=5)# One model call has a 120 second request timeout and one retry on a malformed answer.@@ -89,10 +98,13 @@ class ActivitySummaryTickPlan:emit_embeddings: boolcapture_events: boolstop_reason: str | None- # A default, so a plan that a workflow history recorded before this field existed still decodes.+ # Defaults, so a plan that a workflow history recorded before these fields existed still decodes.limits: RenderLimits = DEFAULT_LIMITSrebuild_before: datetime.datetime | None = Nonemax_versions: int = MAX_VERSIONS+ max_error_rate: float = 1.0+ max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600+ tiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)@frozen@@ -103,11 +115,12 @@ class ActivitySummaryPairInput:min_update_interval_hours: floatemit_embeddings: boolcapture_events: bool- # A default, so a pair input that a tick queued before this field existed still decodes.+ # Defaults, so a pair input that a tick queued before these fields existed still decodes.limits: RenderLimits = DEFAULT_LIMITSrebuild_before: datetime.datetime | None = None# Has a default, so the pair input of a workflow in flight during a deploy still decodes.max_versions: int = MAX_VERSIONS+ max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600@frozen@@ -126,6 +139,7 @@ class ActivitySummaryTickResult:errored: int = 0cost_usd: float = 0.0stop_reason: str | None = None+ tiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPlan:@@ -142,8 +156,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPupdates_left = max(0, config.max_pairs_per_day - usage.updates) if usage else 0cost_left = max(0.0, config.max_cost_usd_per_day - usage.cost_usd) if usage else 0.0tick_limit = limit or config.max_pairs_per_tick- # One more than the cut, to tell a full queue from a queue the cut emptied.- due = select_pairs(config, now, tick_limit + 1) if updates_left and cost_left else []+ selection = select_pairs(config, now, tick_limit) if updates_left and cost_left else None+ pairs = selection.pairs if selection else []+ due_pairs = selection.due_pairs if selection else 0stop_reason: str | None = Noneif gateway_missing:@@ -152,12 +167,12 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPstop_reason = STOP_MAX_PAIRS_PER_DAYelif enabled and not cost_left:stop_reason = STOP_MAX_COST_USD_PER_DAY- elif len(due) > tick_limit:- stop_reason = STOP_MAX_PAIRS_PER_TICK+ elif due_pairs > len(pairs):+ stop_reason = STOP_MAX_PAIRS_PER_TICK if len(pairs) >= tick_limit else STOP_MAX_ORG_SHAREreturn ActivitySummaryTickPlan(enabled=enabled,- pairs=due[:tick_limit],- candidates=len(due),+ pairs=pairs,+ candidates=due_pairs,updates_left=updates_left,cost_left_usd=cost_left,max_update_cost_usd=max_update_cost_usd(config.limits),@@ -167,6 +182,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPemit_embeddings=config.emit_embeddings,capture_events=config.capture_events,limits=config.limits,+ max_error_rate=config.max_error_rate,+ max_skip_backoff_hours=config.max_skip_backoff_hours,+ tiers=selection.tiers if selection else [],stop_reason=stop_reason,rebuild_before=config.rebuild_before,max_versions=config.max_versions,@@ -235,6 +253,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> Alimits=input.limits,rebuild_before=input.rebuild_before,max_versions=input.max_versions,+ max_skip_backoff=datetime.timedelta(hours=input.max_skip_backoff_hours),)called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)except Exception as error:@@ -271,6 +290,7 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:errored=result.errored,cost_usd=result.cost_usd,stop_reason=result.stop_reason,+ tiers=[dataclasses.asdict(tier) for tier in result.tiers],)ph_background_capture()(distinct_id=PIPELINE_DISTINCT_ID,@@ -283,6 +303,11 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:"errored": result.errored,"cost_usd": result.cost_usd,"stop_reason": result.stop_reason,+ **{+ f"{tier.tier}_{name}": getattr(tier, name)+ for tier in result.tiers+ for name in ("due_pairs", "oldest_due_hours", "fresh_share")+ },},)@@ -326,6 +351,9 @@ class ActivitySummaryTickWorkflow:statuses: Counter[str] = Counter()errored = 0cost = 0.0+ # Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.+ calls = 0+ failures = 0updates_left = plan.updates_leftstop_reason = plan.stop_reasonpicked = 0@@ -339,6 +367,10 @@ class ActivitySummaryTickWorkflow:if cost >= plan.cost_left_usd:stop_reason = STOP_MAX_COST_USD_PER_DAYbreak+ # The pairs not attempted yet stay due with no backoff, so the next tick picks them.+ if calls >= _MIN_ERROR_RATE_SAMPLE and failures >= plan.max_error_rate * calls:+ stop_reason = STOP_ERROR_RATE+ breakwave = plan.pairs[index : index + min(concurrency, updates_left)]index += len(wave)picked += len(wave)@@ -356,6 +388,7 @@ class ActivitySummaryTickWorkflow:limits=plan.limits,rebuild_before=plan.rebuild_before,max_versions=plan.max_versions,+ max_skip_backoff_hours=plan.max_skip_backoff_hours,),start_to_close_timeout=_PAIR_TIMEOUT,heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,@@ -369,11 +402,15 @@ class ActivitySummaryTickWorkflow:if isinstance(outcome, BaseException):# The attempt can have called the model, so it counts at the highest cost.errored += 1+ calls += 1+ failures += 1updates_left -= 1cost += plan.max_update_cost_usdcontinuestatuses[outcome.status] += 1if outcome.called_model:+ calls += 1+ failures += outcome.status == UpdateStatus.FAILEDupdates_left -= 1cost += plan.max_update_cost_usd if outcome.cost_usd is None else outcome.cost_usd@@ -385,6 +422,7 @@ class ActivitySummaryTickWorkflow:errored=errored,cost_usd=cost,stop_reason=stop_reason,+ tiers=plan.tiers,)await workflow.execute_activity(capture_activity_summary_tick_activity,diff --git a/products/signals/backend/migrations/0193_useractivitysummary_consecutive_skips.py b/products/signals/backend/migrations/0193_useractivitysummary_consecutive_skips.pynew file mode 100644index 000000000000..4b884bc27680--- /dev/null+++ b/products/signals/backend/migrations/0193_useractivitysummary_consecutive_skips.py@@ -0,0 +1,17 @@+# Generated by Django 5.2.18 on 2026-10-21 22:14++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("signals", "0192_useractivitysummary_queue"),+ ]++ operations = [+ migrations.AddField(+ model_name="useractivitysummary",+ name="consecutive_skips",+ field=models.PositiveSmallIntegerField(default=0),+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex 4699f622c187..7c2c7306e5de 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0192_useractivitysummary_queue+0193_useractivitysummary_consecutive_skipsdiff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex dd31efc4f1ea..7643d1ff9857 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -3864,6 +3864,8 @@ class UserActivitySummary(TeamScopedRootMixin, UUIDModel):# The number of the latest successful update, and of its `UserActivitySummaryVersion` row. 0 before the first.# `db_default` lets a pod on the earlier release insert a row without this column during a deploy.version = models.PositiveIntegerField(default=0, db_default=0)+ # Skips for too few units since the last good update. The skip backoff doubles with each one.+ consecutive_skips = models.PositiveSmallIntegerField(default=0)created_at = models.DateTimeField(auto_now_add=True)updated_at = models.DateTimeField(auto_now=True)@@ -3947,7 +3949,7 @@ class UserActivitySummaryQueueState(TeamScopedRootMixin, UUIDModel):models.UniqueConstraint(fields=["team", "user"], name="signals_activity_queue_team_user"),]indexes = [- # The queue order of `select_pairs`: pairs with no summary, then most pending units, then oldest summary.+ # Covers the pairs with pending units, which are the only pairs `select_pairs` reads.models.Index(models.ExpressionWrapper(models.Q(summarized_at__isnull=False), output_field=models.BooleanField()),models.F("pending_units").desc(),diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.pyindex 9dac59470aeb..99e1cea3c93a 100644--- a/products/signals/backend/test/test_activity_summary_tick.py+++ b/products/signals/backend/test/test_activity_summary_tick.py@@ -29,6 +29,7 @@ from products.signals.backend.activity_summary.config import ActivitySummaryConffrom products.signals.backend.activity_summary.prompt import PROMPT_VERSIONfrom products.signals.backend.activity_summary.queue import (ActivitySummaryPair,+ ActivitySummaryTierStats,DailyUsage,daily_usage,enrolled_team_ids,@@ -43,8 +44,10 @@ from products.signals.backend.activity_summary.schedule import (create_activity_summary_tick_schedule,)from products.signals.backend.activity_summary.tick import (+ STOP_ERROR_RATE,STOP_GATEWAY_NOT_CONFIGURED,STOP_MAX_COST_USD_PER_DAY,+ STOP_MAX_ORG_SHARE,STOP_MAX_PAIRS_PER_DAY,STOP_MAX_PAIRS_PER_TICK,ActivitySummaryPairInput,@@ -93,7 +96,9 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = ["max_versions": -3,"emit_embeddings": "no","rebuild_before": "not a time",+ "max_org_share": 2,"rebuild_budget_share": 2,+ "max_error_rate": 0,},ActivitySummaryConfig(enabled=True, organization_ids=("org-a",), skip_team_ids=(3,)),),@@ -109,6 +114,8 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = ["rebuild_before": "2026-10-01T00:00:00","enroll_all_approved": True,"count_views": True,+ "target_interval_days_free": 7,+ "max_org_share": 0.3,},ActivitySummaryConfig(enabled=True,@@ -120,6 +127,8 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [rebuild_before=datetime.datetime(2026, 10, 1, tzinfo=datetime.UTC),enroll_all_approved=True,count_views=True,+ target_interval_days_free=7.0,+ max_org_share=0.3,),),(@@ -199,33 +208,72 @@ class TestActivitySummaryQueue(BaseTest):):if seed:self._seed(config, tasks)- return [(pair.team_id, pair.user_id) for pair in select_pairs(config or self._config(), now, 10)]+ return [(pair.team_id, pair.user_id) for pair in select_pairs(config or self._config(), now, 10).pairs]- def test_orders_cold_starts_then_most_new_rows_then_oldest_summary(self) -> None:- stale, fresh, busy, cold = (self._user(f"{name}@example.com") for name in ("stale", "fresh", "busy", "cold"))- self._summary(- stale, generated_at=NOW - datetime.timedelta(days=5), input_watermark=NOW - datetime.timedelta(days=3)+ def test_ranks_by_new_units_times_staleness_with_a_cold_start_bonus(self) -> None:+ self.organization.has_active_subscription = True+ self.organization.save()+ cold, recent_busy, old_quiet, recent_quiet = (+ self._user(f"{name}@example.com") for name in ("cold", "recent_busy", "old_quiet", "recent_quiet"))- self._summary(- fresh, generated_at=NOW - datetime.timedelta(days=2), input_watermark=NOW - datetime.timedelta(days=3)+ for user, days, units in ((recent_busy, 1, 20), (old_quiet, 3, 1), (recent_quiet, 1, 2)):+ summarized = NOW - datetime.timedelta(days=days)+ self._summary(user, generated_at=summarized, input_watermark=summarized)+ self._changes(user, units, days_ago=0.5)+ self._changes(cold, 3, days_ago=0.5)++ assert self._select(self._config(min_units=1, max_org_share=1.0)) == [+ (self.team.id, cold.id),+ (self.team.id, recent_busy.id),+ (self.team.id, old_quiet.id),+ (self.team.id, recent_quiet.id),+ ]++ @parameterized.expand([("the whole tick", 1.0, 2, 1), ("half the tick", 0.5, 1, 1)])+ def test_paying_pairs_and_overdue_free_pairs_come_first_and_one_org_takes_at_most_its_share(+ self, _name: str, paying_budget_share: float, paying_picked: int, fresh_picked: int+ ) -> None:+ self.organization.has_active_subscription = True+ self.organization.save()+ free_org = Organization.objects.create(name="Free")+ free_team = Team.objects.create(organization=free_org, name="Free")+ paying_users = [self._user(f"paying{index}@example.com") for index in range(3)]+ overdue, *fresh = (+ User.objects.create_and_join(free_org, f"{name}@example.com", None)+ for name in ("overdue", "free1", "free2"))- self._summary(- busy, generated_at=NOW - datetime.timedelta(days=1), input_watermark=NOW - datetime.timedelta(days=3)+ for user in paying_users:+ self._summary(user, generated_at=NOW - datetime.timedelta(hours=20))+ self._changes(user, 3, days_ago=0.1)+ for user, summarized, due in (+ (overdue, NOW - datetime.timedelta(days=20), NOW - datetime.timedelta(hours=10)),+ *((user, NOW - datetime.timedelta(days=2), None) for user in fresh),+ ):+ UserActivitySummary.objects.for_team(free_team.id).create(+ team_id=free_team.id,+ user_id=user.id,+ status=UserActivitySummary.Status.OK,+ generated_at=summarized,+ next_eligible_at=due,+ )+ self._changes(user, 3, days_ago=0.1, team=free_team)+ config = ActivitySummaryConfig(+ enabled=True, enroll_all_approved=True, max_org_share=0.5, paying_budget_share=paying_budget_share)- self._changes(stale, 1)- self._changes(fresh, 1)- # Rows at or before the watermark are not new.- self._changes(busy, 2, days_ago=4)- self._changes(busy, 2)- self._changes(cold, 1)- tasks = [UserTaskTime(team_id=self.team.id, user_id=busy.id, created_at=NOW - datetime.timedelta(hours=1))]-- assert self._select(self._config(min_units=1), tasks=tasks) == [- (self.team.id, cold.id),- (self.team.id, busy.id),- (self.team.id, stale.id),- (self.team.id, fresh.id),+ self._seed(config)++ selection = select_pairs(config, NOW, 4)++ assert [(pair.team_id, pair.user_id) for pair in selection.pairs] == [+ (free_team.id, overdue.id),+ *((self.team.id, user.id) for user in paying_users[:paying_picked]),+ *((free_team.id, user.id) for user in fresh[:fresh_picked]),]+ assert selection.due_pairs == 6+ assert [+ (tier.tier, tier.due_pairs, round(tier.oldest_due_hours or 0, 3), round(tier.fresh_share or 0, 3))+ for tier in selection.tiers+ ] == [("paying", 3, 0.0, 1.0), ("free", 3, 10.0, round(2 / 3, 3))]@parameterized.expand([("every stale pair fits the share", 0.3, 2), ("the share caps the rebuilds", 0.1, 1)])def test_a_stale_summary_is_due_without_new_rows_and_ranks_with_cold_starts(@@ -246,7 +294,9 @@ class TestActivitySummaryQueue(BaseTest):self._changes(user, 1, days_ago=5)self._changes(busy, 3)- config = self._config(rebuild_before=NOW - datetime.timedelta(days=2, hours=12), rebuild_budget_share=share)+ config = self._config(+ rebuild_before=NOW - datetime.timedelta(days=2, hours=12), rebuild_budget_share=share, max_org_share=1+ )assert self._select(config) == [*[(self.team.id, before_rebuild.id), (self.team.id, old_version.id)][:rebuilds],(self.team.id, busy.id),@@ -312,6 +362,7 @@ class TestActivitySummaryQueue(BaseTest):("the tick cap", {"max_pairs_per_tick": 1}, "phs_test", STOP_MAX_PAIRS_PER_TICK, 1),("the daily pair cap", {"max_pairs_per_day": 1}, "phs_test", STOP_MAX_PAIRS_PER_DAY, 0),("the daily cost cap", {"max_cost_usd_per_day": 0.1}, "phs_test", STOP_MAX_COST_USD_PER_DAY, 0),+ ("the org share", {"max_pairs_per_tick": 2, "max_org_share": 0.5}, "phs_test", STOP_MAX_ORG_SHARE, 1),("no cap", {}, "phs_test", None, 2),("no gateway", {}, "", STOP_GATEWAY_NOT_CONFIGURED, 0),]@@ -521,13 +572,15 @@ def _plan(pairs: int, **overrides: Any) -> ActivitySummaryTickPlan:"emit_embeddings": True,"capture_events": True,"limits": DEFAULT_LIMITS,+ "max_error_rate": 0.5,+ "tiers": [ActivitySummaryTierStats(tier="paying", due_pairs=pairs)],"stop_reason": None,}return ActivitySummaryTickPlan(**{**fields, **overrides})async def _run_tick(- plan: ActivitySummaryTickPlan, pair_cost: float | None+ plan: ActivitySummaryTickPlan, pair_cost: float | None, pair_status: str = "ok") -> tuple[ActivitySummaryTickResult, int, list[ActivitySummaryTickResult]]:running = peak = 0captured: list[ActivitySummaryTickResult] = []@@ -543,7 +596,7 @@ async def _run_tick(peak = max(peak, running)await asyncio.sleep(0.01)running -= 1- return ActivitySummaryPairResult(status="ok", cost_usd=pair_cost, called_model=True)+ return ActivitySummaryPairResult(status=pair_status, cost_usd=pair_cost, called_model=True)@activity.defn(name="capture_activity_summary_tick_activity")async def fake_capture(result: ActivitySummaryTickResult) -> None:@@ -568,21 +621,24 @@ async def _run_tick(@pytest.mark.asyncio@pytest.mark.parametrize(- "plan,pair_cost,picked,stop_reason",+ "plan,pair_cost,pair_status,picked,stop_reason",[- (_plan(5), 0.01, 5, None),- (_plan(5, updates_left=3), 0.01, 3, STOP_MAX_PAIRS_PER_DAY),+ (_plan(5), 0.01, "ok", 5, None),+ (_plan(5, updates_left=3), 0.01, "ok", 3, STOP_MAX_PAIRS_PER_DAY),# A missing cost counts at the bound, so the second wave spends the last of the budget.- (_plan(6, cost_left_usd=3.0), None, 4, STOP_MAX_COST_USD_PER_DAY),+ (_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY),+ # The breaker waits for a sample of 10 model calls, which is five waves.+ (_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE),],)-async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps(- plan: ActivitySummaryTickPlan, pair_cost: float | None, picked: int, stop_reason: str | None+async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps_and_the_error_rate(+ plan: ActivitySummaryTickPlan, pair_cost: float | None, pair_status: str, picked: int, stop_reason: str | None) -> None:- result, peak, captured = await _run_tick(plan, pair_cost)+ result, peak, captured = await _run_tick(plan, pair_cost, pair_status)assert peak <= plan.max_concurrency- assert (result.picked, result.statuses, result.stop_reason) == (picked, {"ok": picked}, stop_reason)+ assert (result.picked, result.statuses, result.stop_reason) == (picked, {pair_status: picked}, stop_reason)+ assert result.tiers == plan.tiersassert captured == [result]diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.pyindex a94e8f9f473c..67ff9e3618fd 100644--- a/products/signals/backend/test/test_activity_summary_update.py+++ b/products/signals/backend/test/test_activity_summary_update.py@@ -271,8 +271,13 @@ class TestUpdateActivitySummary(BaseTest):assert skipped is not Noneassert (skipped.status, skipped.summary) == (UserActivitySummary.Status.SKIPPED, None)assert skipped.next_eligible_at == NOW + datetime.timedelta(hours=6)-with time_machine.travel(NOW + datetime.timedelta(hours=7), tick=False):+ assert self._update(llm).status == UpdateStatus.SKIPPED_TOO_FEW_UNITS+ skipped_again = self._stored()+ assert skipped_again is not None+ assert skipped_again.next_eligible_at == NOW + datetime.timedelta(hours=7 + 12)++ with time_machine.travel(NOW + datetime.timedelta(hours=20), tick=False):assert self._update(FakeLLM([ANSWER]), min_units=2).status == UpdateStatus.OKbefore = self._stored()assert before is not None