Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): prioritize activity summary pairs by tier, activity and staleness (#121717)

posthog[bot]c16f9819811e
diff --git a/products/signals/backend/activity_summary/config.py b/products/signals/backend/activity_summary/config.py
index bf3b7a301405..b4f710115980 100644
--- a/products/signals/backend/activity_summary/config.py
+++ b/products/signals/backend/activity_summary/config.py
@@ -14,7 +14,12 @@ import structlog
from posthog.dataclasses import frozen
-from products.signals.backend.activity_summary.logic import MAX_VERSIONS, MIN_UNITS, MIN_UPDATE_INTERVAL
+from products.signals.backend.activity_summary.logic import (
+ MAX_SKIP_BACKOFF,
+ MAX_VERSIONS,
+ MIN_UNITS,
+ MIN_UPDATE_INTERVAL,
+)
from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, RenderLimits
from products.signals.backend.scout_harness.team_limits import read_flag_payload
@@ -44,6 +49,20 @@ class ActivitySummaryConfig:
capture_events: bool = True
# Count viewed objects in the queue. Turn it on only when the update reads views as an input.
count_views: bool = False
+ # The summary age each tier aims for. A pair's staleness is its summary age over this interval.
+ target_interval_days_paying: float = 3.0
+ target_interval_days_free: float = 5.0
+ # A free pair with at least this staleness joins the paying pass, so paying pairs cannot starve it.
+ max_overdue_factor: float = 3.0
+ # Added to the score of a pair with no summary, because a first summary changes the most.
+ cold_start_bonus: float = 5.0
+ # The share of the tick that the paying pass can take. The free pass gets the rest.
+ paying_budget_share: float = 1.0
+ # The largest share of one pass that the pairs of one organization can take.
+ max_org_share: float = 0.1
+ # The tick stops when this share of its model calls failed or raised.
+ max_error_rate: float = 0.5
+ max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600
limits: RenderLimits = DEFAULT_LIMITS
# Summaries generated before this time are rebuilt from the full window, as after a version change.
rebuild_before: datetime.datetime | None = None
@@ -132,6 +151,18 @@ def parse_config(payload: Any) -> ActivitySummaryConfig:
emit_embeddings=_bool(payload.get("emit_embeddings"), defaults.emit_embeddings),
capture_events=_bool(payload.get("capture_events"), defaults.capture_events),
count_views=_bool(payload.get("count_views"), defaults.count_views),
+ target_interval_days_paying=_positive_number(
+ payload.get("target_interval_days_paying"), defaults.target_interval_days_paying
+ ),
+ target_interval_days_free=_positive_number(
+ payload.get("target_interval_days_free"), defaults.target_interval_days_free
+ ),
+ max_overdue_factor=_positive_number(payload.get("max_overdue_factor"), defaults.max_overdue_factor),
+ cold_start_bonus=_positive_number(payload.get("cold_start_bonus"), defaults.cold_start_bonus),
+ paying_budget_share=_fraction(payload.get("paying_budget_share"), defaults.paying_budget_share),
+ max_org_share=_fraction(payload.get("max_org_share"), defaults.max_org_share),
+ max_error_rate=_fraction(payload.get("max_error_rate"), defaults.max_error_rate),
+ max_skip_backoff_hours=_positive_number(payload.get("max_skip_backoff_hours"), defaults.max_skip_backoff_hours),
limits=parse_limits(payload.get("limits")),
rebuild_before=_timestamp(payload.get("rebuild_before")),
rebuild_budget_share=_fraction(payload.get("rebuild_budget_share"), defaults.rebuild_budget_share),
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index 1dbc3bb8a868..3a6ed5d72db8 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -37,7 +37,11 @@ MIN_UPDATE_INTERVAL = datetime.timedelta(days=1)
# The `UserActivitySummaryVersion` rows kept per summary. An update deletes the older ones.
MAX_VERSIONS = 30
# A pair that skipped or failed waits this long, so it does not take the queue head on every tick.
+# Each further skip since the last good update doubles the wait, up to the `max_skip_backoff` of the update.
RETRY_BACKOFF = datetime.timedelta(hours=6)
+MAX_SKIP_BACKOFF = datetime.timedelta(hours=72)
+# Enough doublings to pass any sane maximum, and a small enough exponent for `timedelta`.
+_MAX_SKIP_DOUBLINGS = 16
# The error prefixes of an update that raised before it stored a result, before and after the model call.
EXCEPTION_ERROR_PREFIX = "exception: "
EXCEPTION_AFTER_MODEL_CALL_ERROR_PREFIX = "exception_after_model_call: "
@@ -99,22 +103,39 @@ def is_stale(stored: StoredSummaryVersion | None, rebuild_before: datetime.datet
return rebuild_before is not None and stored.generated_at < rebuild_before
+def skip_backoff(consecutive_skips: int, max_skip_backoff: datetime.timedelta) -> datetime.timedelta:
+ """The wait after a skip for too few units, when `consecutive_skips` skips came before it."""
+ return max(RETRY_BACKOFF, min(RETRY_BACKOFF * 2 ** min(consecutive_skips, _MAX_SKIP_DOUBLINGS), max_skip_backoff))
+
+
def _input_start(watermark: datetime.datetime | None, now: datetime.datetime, window_days: int) -> datetime.datetime:
earliest = now - datetime.timedelta(days=window_days)
return earliest if watermark is None else max(earliest, watermark + _WATERMARK_STEP)
def _back_off(
- team_id: int, user_id: int, stored: UserActivitySummary | None, next_eligible_at: datetime.datetime
+ team_id: int,
+ user_id: int,
+ stored: UserActivitySummary | None,
+ now: datetime.datetime,
+ max_skip_backoff: datetime.timedelta,
) -> None:
+ skips = stored.consecutive_skips if stored is not None else 0
+ next_eligible_at = now + skip_backoff(skips, max_skip_backoff)
# `update()` leaves `updated_at` alone, so the daily caps do not count a skip as a model call.
if stored is not None:
- UserActivitySummary.objects.for_team(team_id).filter(id=stored.id).update(next_eligible_at=next_eligible_at)
+ UserActivitySummary.objects.for_team(team_id).filter(id=stored.id).update(
+ next_eligible_at=next_eligible_at, consecutive_skips=min(skips + 1, _MAX_SKIP_DOUBLINGS)
+ )
return
UserActivitySummary.objects.for_team(team_id).get_or_create(
team_id=team_id,
user_id=user_id,
- defaults={"status": UserActivitySummary.Status.SKIPPED, "next_eligible_at": next_eligible_at},
+ defaults={
+ "status": UserActivitySummary.Status.SKIPPED,
+ "next_eligible_at": next_eligible_at,
+ "consecutive_skips": 1,
+ },
)
@@ -218,6 +239,7 @@ def update_activity_summary(
limits: RenderLimits = DEFAULT_LIMITS,
rebuild_before: datetime.datetime | None = None,
max_versions: int = MAX_VERSIONS,
+ max_skip_backoff: datetime.timedelta = MAX_SKIP_BACKOFF,
) -> ActivitySummaryUpdate:
"""Update the stored activity summary of one user in one project.
@@ -225,7 +247,8 @@ def update_activity_summary(
run or for a stale summary (see `is_stale`). A stale summary is not sent to the model. A failed
update records the error and keeps the stored summary and the watermark. With `dry_run`, the
model is still called but nothing is written. `force` ignores `next_eligible_at`.
- A skip for too few units and a failed update set `next_eligible_at` to `RETRY_BACKOFF` from now.
+ A failed update sets `next_eligible_at` to `RETRY_BACKOFF` from now. A skip for too few units sets it
+ to `skip_backoff` from now, which doubles with each skip until a good update resets it.
A successful update also appends a `UserActivitySummaryVersion` and keeps the latest `max_versions`.
"""
team = Team.objects.select_related("organization").get(id=team_id)
@@ -253,7 +276,7 @@ def update_activity_summary(
)
if rendered is None or rendered.units < min_units:
if not dry_run:
- _back_off(team_id, user_id, stored, now + RETRY_BACKOFF)
+ _back_off(team_id, user_id, stored, now, max_skip_backoff)
return ActivitySummaryUpdate(
status=UpdateStatus.SKIPPED_TOO_FEW_UNITS,
rendered_input=rendered.text if rendered else None,
@@ -332,6 +355,7 @@ def update_activity_summary(
"render_version": RENDER_VERSION,
"input_watermark": watermark,
"next_eligible_at": now + min_update_interval,
+ "consecutive_skips": 0,
},
max_versions,
)
diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.py
index a1d2752e9bd5..1ed5def686db 100644
--- a/products/signals/backend/activity_summary/queue.py
+++ b/products/signals/backend/activity_summary/queue.py
@@ -1,9 +1,10 @@
"""Pick the (team, user) pairs that the next activity summary tick updates, and measure the daily caps.
Each pair has a queue state row. One cursor per input source reads the new rows of all enrolled teams
-and adds them to the pending units of their pair. `select_pairs` then reads the due pairs in one
-ordered query. Row counts here are only a queue estimate. `update_activity_summary` still applies the
-real drop rules and `min_units`, and backs a pair off when it skips, so the pair leaves the queue head.
+and adds them to the pending units of their pair. `select_pairs` then ranks the due pairs and fills the
+tick budget in two passes: paying organizations first, then free organizations with the budget that is
+left. Row counts here are only a queue estimate. `update_activity_summary` still applies the real drop
+rules and `min_units`, and backs a pair off when it skips, so the pair leaves the queue head.
"""
import math
@@ -12,8 +13,25 @@ from collections.abc import Callable
from typing import Any
from django.db import transaction
-from django.db.models import BooleanField, Case, Count, ExpressionWrapper, F, Max, OuterRef, Q, Subquery, Sum, When
-from django.db.models.functions import Coalesce, Greatest
+from django.db.models import (
+ BooleanField,
+ Case,
+ Count,
+ DateTimeField,
+ F,
+ FloatField,
+ Max,
+ OuterRef,
+ Q,
+ QuerySet,
+ Subquery,
+ Sum,
+ Value,
+ When,
+ Window,
+)
+from django.db.models.expressions import Combinable
+from django.db.models.functions import Cast, Coalesce, Extract, Greatest, Ln, RowNumber
from posthog.dataclasses import frozen
from posthog.models import ActivityLog, FileSystemViewLog, Team
@@ -54,14 +72,10 @@ SOURCE_VIEWS = "views"
FEED_LAG = datetime.timedelta(minutes=5)
# One tick reads at most this much of one source. A cursor that fell behind catches up over the next ticks.
MAX_FEED_STEP = datetime.timedelta(hours=6)
-# Pairs with no summary first, then the most pending units, then the oldest summary. The state index has this order.
-_QUEUE_ORDER = (
- ExpressionWrapper(Q(summarized_at__isnull=False), output_field=BooleanField()).asc(),
- F("pending_units").desc(),
- F("summarized_at").asc(),
- "team_id",
- "user_id",
-)
+TIER_PAYING = "paying"
+TIER_FREE = "free"
+# A summary younger than this many target intervals counts as fresh on the tick event.
+_FRESH_STALENESS = 2
@frozen(order=True)
@@ -70,6 +84,23 @@ class ActivitySummaryPair:
user_id: int
+@frozen
+class ActivitySummaryTierStats:
+ tier: str
+ due_pairs: int = 0
+ # Hours since the due time of the pair that waits longest. Null when no pair is due.
+ oldest_due_hours: float | None = None
+ # The share of pairs with activity in the window whose summary is younger than twice the tier's target interval.
+ fresh_share: float | None = None
+
+
+@frozen
+class PairSelection:
+ pairs: list[ActivitySummaryPair]
+ due_pairs: int
+ tiers: list[ActivitySummaryTierStats]
+
+
@frozen
class _NewRows:
units: int
@@ -372,41 +403,139 @@ def _current(config: ActivitySummaryConfig) -> Q:
return Q(summarized_at__isnull=True) | current
-def select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: int) -> list[ActivitySummaryPair]:
+def _hours_since(now: datetime.datetime, at: Coalesce) -> Combinable:
+ age = Value(now, output_field=DateTimeField()) - at
+ return Cast(Extract(age, "epoch"), FloatField()) / Value(3600.0)
+
+
+def _states(config: ActivitySummaryConfig, now: datetime.datetime, team_ids: list[int]) -> QuerySet:
+ """The queue states of these teams, with their tier, staleness and priority score.
+
+ Staleness is the summary age over the tier's target interval. A pair with no summary counts its age
+ from its first queued row. The score is log(1 + pending units) times staleness, plus a bonus for a
+ pair with no summary, or a bonus plus staleness for a stale summary.
+ """
+ paying = Coalesce(F("team__organization__has_active_subscription"), Value(False))
+ target_hours = Case(
+ When(paying=True, then=Value(config.target_interval_days_paying * 24)),
+ default=Value(config.target_interval_days_free * 24),
+ output_field=FloatField(),
+ )
+ age_hours = _hours_since(now, Coalesce("summarized_at", "created_at"))
+ stale = Case(When(_current(config), then=Value(False)), default=Value(True), output_field=BooleanField())
+ cold_bonus = Case(
+ When(summarized_at__isnull=True, then=Value(config.cold_start_bonus)),
+ # A stale pair often has no pending units, so its staleness orders it among the other rebuilds.
+ When(stale=True, then=Value(config.cold_start_bonus) + F("staleness")),
+ default=Value(0.0),
+ output_field=FloatField(),
+ )
+ return (
+ # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)
+ UserActivitySummaryQueueState.all_teams.filter(team_id__in=team_ids)
+ .annotate(organization_id=F("team__organization_id"), paying=paying, stale=stale)
+ .annotate(staleness=age_hours / target_hours)
+ .annotate(score=Ln(F("pending_units") + 1) * F("staleness") + cold_bonus)
+ )
+
+
+def _due(states: QuerySet, config: ActivitySummaryConfig, now: datetime.datetime) -> QuerySet:
+ return states.filter(
+ Q(due_at__isnull=True) | Q(due_at__lte=now),
+ Q(pending_units__gte=max(1, config.min_units)) | Q(stale=True),
+ last_activity_at__gte=now - datetime.timedelta(days=config.limits.window_days),
+ )
+
+
+def _cap_rebuilds(due: QuerySet, max_rebuilds: int) -> QuerySet:
+ """The due pairs with at most `max_rebuilds` stale pairs, paying pairs and the highest scores first."""
+ rebuild_ids = list(
+ due.filter(stale=True)
+ .order_by("-paying", F("score").desc(), "team_id", "user_id")
+ .values_list("id", flat=True)[:max_rebuilds]
+ )
+ return due.filter(Q(stale=False) | Q(id__in=rebuild_ids))
+
+
+def _fill_pass(due: QuerySet, limit: int, max_org_share: float) -> list[UserActivitySummaryQueueState]:
+ """The highest scores first, with at most `max_org_share` of `limit` from one organization."""
+ if limit <= 0:
+ return []
+ order = (F("score").desc(), "team_id", "user_id")
+ return list(
+ due.annotate(org_rank=Window(RowNumber(), partition_by=[F("organization_id")], order_by=order))
+ .filter(org_rank__lte=max(1, int(max_org_share * limit)))
+ .order_by(*order)
+ .only("id", "team_id", "user_id")[:limit]
+ )
+
+
+def _tier_stats(
+ states: QuerySet, due: QuerySet, config: ActivitySummaryConfig, now: datetime.datetime
+) -> list[ActivitySummaryTierStats]:
+ window_start = now - datetime.timedelta(days=config.limits.window_days)
+ waiting = {
+ row["paying"]: row
+ for row in due.values("paying").annotate(
+ pairs=Count("id"),
+ oldest=Max(_hours_since(now, Coalesce("due_at", "created_at"))),
+ )
+ }
+ activity = {
+ row["paying"]: row
+ for row in states.filter(last_activity_at__gte=window_start)
+ .values("paying")
+ .annotate(
+ active=Count("id"),
+ fresh=Count("id", filter=Q(summarized_at__isnull=False, staleness__lt=_FRESH_STALENESS)),
+ )
+ }
+ stats = []
+ for tier, paying in ((TIER_PAYING, True), (TIER_FREE, False)):
+ due_row = waiting.get(paying)
+ activity_row = activity.get(paying)
+ stats.append(
+ ActivitySummaryTierStats(
+ tier=tier,
+ due_pairs=due_row["pairs"] if due_row else 0,
+ oldest_due_hours=max(0.0, due_row["oldest"]) if due_row else None,
+ fresh_share=activity_row["fresh"] / activity_row["active"] if activity_row else None,
+ )
+ )
+ return stats
+
+
+def select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: int) -> PairSelection:
"""The due pairs of the enrolled teams, highest priority first, cut to `limit`.
A pair is due when it has a row in the last `window_days` and its due time is empty or past. A pair
with a current summary also needs at least `min_units` pending units. One unit never comes from less
than one row, so a pair with fewer rows would only skip. A pair with a stale summary needs no pending
- units, because its update reads the full window. Stale pairs take at most `rebuild_budget_share` of
- `limit`, so a version change does not stop the other updates.
-
- Pairs with no summary come first, then stale pairs with the oldest summary first, then the pairs with
- the most pending units, then the oldest summaries.
+ units, because its update reads the full window, and it ranks with the pairs with no summary.
+ Stale pairs take at most `rebuild_budget_share` of `limit`, paying pairs first, so a version change
+ does not stop the other updates. The first pass ranks the paying pairs and the free pairs with at
+ least `max_overdue_factor` staleness, for `paying_budget_share` of `limit`. The second pass ranks the
+ other free pairs for the budget that is left.
"""
team_ids = enrolled_team_ids(config)
- if not team_ids or limit <= 0:
- return []
- window_start = now - datetime.timedelta(days=config.limits.window_days)
- due = (
- # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)
- UserActivitySummaryQueueState.all_teams.filter(team_id__in=team_ids, last_activity_at__gte=window_start).filter(
- Q(due_at__isnull=True) | Q(due_at__lte=now)
- )
+ if not team_ids:
+ return PairSelection(pairs=[], due_pairs=0, tiers=[])
+ states = _states(config, now, team_ids)
+ due = _due(states, config, now)
+ tiers = _tier_stats(states, due, config, now)
+ due = _cap_rebuilds(due, math.ceil(limit * config.rebuild_budget_share))
+ first = _fill_pass(
+ due.filter(Q(paying=True) | Q(staleness__gte=config.max_overdue_factor)),
+ int(limit * config.paying_budget_share),
+ config.max_org_share,
)
- current = _current(config)
- updates = list(
- due.filter(current, pending_units__gte=max(1, config.min_units))
- .order_by(*_QUEUE_ORDER)
- .values_list("team_id", "user_id", "summarized_at")[:limit]
+ second = _fill_pass(
+ due.filter(paying=False).exclude(id__in=[state.id for state in first]),
+ limit - len(first),
+ config.max_org_share,
)
- rebuilds = list(
- due.exclude(current)
- .order_by("summarized_at", "team_id", "user_id")
- .values_list("team_id", "user_id")[: math.ceil(limit * config.rebuild_budget_share)]
+ return PairSelection(
+ pairs=[ActivitySummaryPair(team_id=state.team_id, user_id=state.user_id) for state in first + second],
+ due_pairs=sum(tier.due_pairs for tier in tiers),
+ tiers=tiers,
)
- cold = [(team_id, user_id) for team_id, user_id, at in updates if at is None]
- warm = [(team_id, user_id) for team_id, user_id, at in updates if at is not None]
- return [ActivitySummaryPair(team_id=team_id, user_id=user_id) for team_id, user_id in cold + rebuilds + warm][
- :limit
- ]
diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.py
index 4a535621749e..b3ea7973a074 100644
--- a/products/signals/backend/activity_summary/tick.py
+++ b/products/signals/backend/activity_summary/tick.py
@@ -6,6 +6,7 @@ tick off. The pair activity calls `update_activity_summary`, then queues the emb
import asyncio
import datetime
+import dataclasses
from collections import Counter
from django.conf import settings
@@ -27,6 +28,7 @@ from posthog.temporal.common.utils import close_db_connections
from products.signals.backend.activity_summary.config import read_config
from products.signals.backend.activity_summary.logic import (
+ MAX_SKIP_BACKOFF,
MAX_VERSIONS,
ActivitySummaryUpdate,
UpdateAfterModelCallError,
@@ -37,6 +39,7 @@ from products.signals.backend.activity_summary.logic import (
from products.signals.backend.activity_summary.prompt import PROMPT_VERSION
from products.signals.backend.activity_summary.queue import (
ActivitySummaryPair,
+ ActivitySummaryTierStats,
daily_usage,
feed_queue,
max_update_cost_usd,
@@ -59,6 +62,12 @@ STOP_MAX_PAIRS_PER_TICK = "max_pairs_per_tick"
STOP_MAX_PAIRS_PER_DAY = "max_pairs_per_day"
STOP_MAX_COST_USD_PER_DAY = "max_cost_usd_per_day"
STOP_GATEWAY_NOT_CONFIGURED = "gateway_not_configured"
+# Due pairs are left, but each organization with due pairs already has its share of the tick.
+STOP_MAX_ORG_SHARE = "max_org_share"
+STOP_ERROR_RATE = "error_rate"
+
+# The circuit breaker waits for this many model calls, so a few early failures do not stop the tick.
+_MIN_ERROR_RATE_SAMPLE = 10
_PLAN_TIMEOUT = datetime.timedelta(minutes=5)
# One model call has a 120 second request timeout and one retry on a malformed answer.
@@ -89,10 +98,13 @@ class ActivitySummaryTickPlan:
emit_embeddings: bool
capture_events: bool
stop_reason: str | None
- # A default, so a plan that a workflow history recorded before this field existed still decodes.
+ # Defaults, so a plan that a workflow history recorded before these fields existed still decodes.
limits: RenderLimits = DEFAULT_LIMITS
rebuild_before: datetime.datetime | None = None
max_versions: int = MAX_VERSIONS
+ max_error_rate: float = 1.0
+ max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600
+ tiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)
@frozen
@@ -103,11 +115,12 @@ class ActivitySummaryPairInput:
min_update_interval_hours: float
emit_embeddings: bool
capture_events: bool
- # A default, so a pair input that a tick queued before this field existed still decodes.
+ # Defaults, so a pair input that a tick queued before these fields existed still decodes.
limits: RenderLimits = DEFAULT_LIMITS
rebuild_before: datetime.datetime | None = None
# Has a default, so the pair input of a workflow in flight during a deploy still decodes.
max_versions: int = MAX_VERSIONS
+ max_skip_backoff_hours: float = MAX_SKIP_BACKOFF.total_seconds() / 3600
@frozen
@@ -126,6 +139,7 @@ class ActivitySummaryTickResult:
errored: int = 0
cost_usd: float = 0.0
stop_reason: str | None = None
+ tiers: list[ActivitySummaryTierStats] = dataclasses.field(default_factory=list)
def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPlan:
@@ -142,8 +156,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
updates_left = max(0, config.max_pairs_per_day - usage.updates) if usage else 0
cost_left = max(0.0, config.max_cost_usd_per_day - usage.cost_usd) if usage else 0.0
tick_limit = limit or config.max_pairs_per_tick
- # One more than the cut, to tell a full queue from a queue the cut emptied.
- due = select_pairs(config, now, tick_limit + 1) if updates_left and cost_left else []
+ selection = select_pairs(config, now, tick_limit) if updates_left and cost_left else None
+ pairs = selection.pairs if selection else []
+ due_pairs = selection.due_pairs if selection else 0
stop_reason: str | None = None
if gateway_missing:
@@ -152,12 +167,12 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
stop_reason = STOP_MAX_PAIRS_PER_DAY
elif enabled and not cost_left:
stop_reason = STOP_MAX_COST_USD_PER_DAY
- elif len(due) > tick_limit:
- stop_reason = STOP_MAX_PAIRS_PER_TICK
+ elif due_pairs > len(pairs):
+ stop_reason = STOP_MAX_PAIRS_PER_TICK if len(pairs) >= tick_limit else STOP_MAX_ORG_SHARE
return ActivitySummaryTickPlan(
enabled=enabled,
- pairs=due[:tick_limit],
- candidates=len(due),
+ pairs=pairs,
+ candidates=due_pairs,
updates_left=updates_left,
cost_left_usd=cost_left,
max_update_cost_usd=max_update_cost_usd(config.limits),
@@ -167,6 +182,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
emit_embeddings=config.emit_embeddings,
capture_events=config.capture_events,
limits=config.limits,
+ max_error_rate=config.max_error_rate,
+ max_skip_backoff_hours=config.max_skip_backoff_hours,
+ tiers=selection.tiers if selection else [],
stop_reason=stop_reason,
rebuild_before=config.rebuild_before,
max_versions=config.max_versions,
@@ -235,6 +253,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A
limits=input.limits,
rebuild_before=input.rebuild_before,
max_versions=input.max_versions,
+ max_skip_backoff=datetime.timedelta(hours=input.max_skip_backoff_hours),
)
called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)
except Exception as error:
@@ -271,6 +290,7 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:
errored=result.errored,
cost_usd=result.cost_usd,
stop_reason=result.stop_reason,
+ tiers=[dataclasses.asdict(tier) for tier in result.tiers],
)
ph_background_capture()(
distinct_id=PIPELINE_DISTINCT_ID,
@@ -283,6 +303,11 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:
"errored": result.errored,
"cost_usd": result.cost_usd,
"stop_reason": result.stop_reason,
+ **{
+ f"{tier.tier}_{name}": getattr(tier, name)
+ for tier in result.tiers
+ for name in ("due_pairs", "oldest_due_hours", "fresh_share")
+ },
},
)
@@ -326,6 +351,9 @@ class ActivitySummaryTickWorkflow:
statuses: Counter[str] = Counter()
errored = 0
cost = 0.0
+ # Model calls of this tick and the calls among them that failed or raised, for the circuit breaker.
+ calls = 0
+ failures = 0
updates_left = plan.updates_left
stop_reason = plan.stop_reason
picked = 0
@@ -339,6 +367,10 @@ class ActivitySummaryTickWorkflow:
if cost >= plan.cost_left_usd:
stop_reason = STOP_MAX_COST_USD_PER_DAY
break
+ # The pairs not attempted yet stay due with no backoff, so the next tick picks them.
+ if calls >= _MIN_ERROR_RATE_SAMPLE and failures >= plan.max_error_rate * calls:
+ stop_reason = STOP_ERROR_RATE
+ break
wave = plan.pairs[index : index + min(concurrency, updates_left)]
index += len(wave)
picked += len(wave)
@@ -356,6 +388,7 @@ class ActivitySummaryTickWorkflow:
limits=plan.limits,
rebuild_before=plan.rebuild_before,
max_versions=plan.max_versions,
+ max_skip_backoff_hours=plan.max_skip_backoff_hours,
),
start_to_close_timeout=_PAIR_TIMEOUT,
heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,
@@ -369,11 +402,15 @@ class ActivitySummaryTickWorkflow:
if isinstance(outcome, BaseException):
# The attempt can have called the model, so it counts at the highest cost.
errored += 1
+ calls += 1
+ failures += 1
updates_left -= 1
cost += plan.max_update_cost_usd
continue
statuses[outcome.status] += 1
if outcome.called_model:
+ calls += 1
+ failures += outcome.status == UpdateStatus.FAILED
updates_left -= 1
cost += plan.max_update_cost_usd if outcome.cost_usd is None else outcome.cost_usd
@@ -385,6 +422,7 @@ class ActivitySummaryTickWorkflow:
errored=errored,
cost_usd=cost,
stop_reason=stop_reason,
+ tiers=plan.tiers,
)
await workflow.execute_activity(
capture_activity_summary_tick_activity,
diff --git a/products/signals/backend/migrations/0193_useractivitysummary_consecutive_skips.py b/products/signals/backend/migrations/0193_useractivitysummary_consecutive_skips.py
new file mode 100644
index 000000000000..4b884bc27680
--- /dev/null
+++ b/products/signals/backend/migrations/0193_useractivitysummary_consecutive_skips.py
@@ -0,0 +1,17 @@
+# Generated by Django 5.2.18 on 2026-10-21 22:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0192_useractivitysummary_queue"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="useractivitysummary",
+ name="consecutive_skips",
+ field=models.PositiveSmallIntegerField(default=0),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 4699f622c187..7c2c7306e5de 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0192_useractivitysummary_queue
+0193_useractivitysummary_consecutive_skips
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index dd31efc4f1ea..7643d1ff9857 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -3864,6 +3864,8 @@ class UserActivitySummary(TeamScopedRootMixin, UUIDModel):
# The number of the latest successful update, and of its `UserActivitySummaryVersion` row. 0 before the first.
# `db_default` lets a pod on the earlier release insert a row without this column during a deploy.
version = models.PositiveIntegerField(default=0, db_default=0)
+ # Skips for too few units since the last good update. The skip backoff doubles with each one.
+ consecutive_skips = models.PositiveSmallIntegerField(default=0)
created_at = models.DateTimeField(auto_now_add=True)
updated_at = models.DateTimeField(auto_now=True)
@@ -3947,7 +3949,7 @@ class UserActivitySummaryQueueState(TeamScopedRootMixin, UUIDModel):
models.UniqueConstraint(fields=["team", "user"], name="signals_activity_queue_team_user"),
]
indexes = [
- # The queue order of `select_pairs`: pairs with no summary, then most pending units, then oldest summary.
+ # Covers the pairs with pending units, which are the only pairs `select_pairs` reads.
models.Index(
models.ExpressionWrapper(models.Q(summarized_at__isnull=False), output_field=models.BooleanField()),
models.F("pending_units").desc(),
diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.py
index 9dac59470aeb..99e1cea3c93a 100644
--- a/products/signals/backend/test/test_activity_summary_tick.py
+++ b/products/signals/backend/test/test_activity_summary_tick.py
@@ -29,6 +29,7 @@ from products.signals.backend.activity_summary.config import ActivitySummaryConf
from products.signals.backend.activity_summary.prompt import PROMPT_VERSION
from products.signals.backend.activity_summary.queue import (
ActivitySummaryPair,
+ ActivitySummaryTierStats,
DailyUsage,
daily_usage,
enrolled_team_ids,
@@ -43,8 +44,10 @@ from products.signals.backend.activity_summary.schedule import (
create_activity_summary_tick_schedule,
)
from products.signals.backend.activity_summary.tick import (
+ STOP_ERROR_RATE,
STOP_GATEWAY_NOT_CONFIGURED,
STOP_MAX_COST_USD_PER_DAY,
+ STOP_MAX_ORG_SHARE,
STOP_MAX_PAIRS_PER_DAY,
STOP_MAX_PAIRS_PER_TICK,
ActivitySummaryPairInput,
@@ -93,7 +96,9 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
"max_versions": -3,
"emit_embeddings": "no",
"rebuild_before": "not a time",
+ "max_org_share": 2,
"rebuild_budget_share": 2,
+ "max_error_rate": 0,
},
ActivitySummaryConfig(enabled=True, organization_ids=("org-a",), skip_team_ids=(3,)),
),
@@ -109,6 +114,8 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
"rebuild_before": "2026-10-01T00:00:00",
"enroll_all_approved": True,
"count_views": True,
+ "target_interval_days_free": 7,
+ "max_org_share": 0.3,
},
ActivitySummaryConfig(
enabled=True,
@@ -120,6 +127,8 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
rebuild_before=datetime.datetime(2026, 10, 1, tzinfo=datetime.UTC),
enroll_all_approved=True,
count_views=True,
+ target_interval_days_free=7.0,
+ max_org_share=0.3,
),
),
(
@@ -199,33 +208,72 @@ class TestActivitySummaryQueue(BaseTest):
):
if seed:
self._seed(config, tasks)
- return [(pair.team_id, pair.user_id) for pair in select_pairs(config or self._config(), now, 10)]
+ return [(pair.team_id, pair.user_id) for pair in select_pairs(config or self._config(), now, 10).pairs]
- def test_orders_cold_starts_then_most_new_rows_then_oldest_summary(self) -> None:
- stale, fresh, busy, cold = (self._user(f"{name}@example.com") for name in ("stale", "fresh", "busy", "cold"))
- self._summary(
- stale, generated_at=NOW - datetime.timedelta(days=5), input_watermark=NOW - datetime.timedelta(days=3)
+ def test_ranks_by_new_units_times_staleness_with_a_cold_start_bonus(self) -> None:
+ self.organization.has_active_subscription = True
+ self.organization.save()
+ cold, recent_busy, old_quiet, recent_quiet = (
+ self._user(f"{name}@example.com") for name in ("cold", "recent_busy", "old_quiet", "recent_quiet")
)
- self._summary(
- fresh, generated_at=NOW - datetime.timedelta(days=2), input_watermark=NOW - datetime.timedelta(days=3)
+ for user, days, units in ((recent_busy, 1, 20), (old_quiet, 3, 1), (recent_quiet, 1, 2)):
+ summarized = NOW - datetime.timedelta(days=days)
+ self._summary(user, generated_at=summarized, input_watermark=summarized)
+ self._changes(user, units, days_ago=0.5)
+ self._changes(cold, 3, days_ago=0.5)
+
+ assert self._select(self._config(min_units=1, max_org_share=1.0)) == [
+ (self.team.id, cold.id),
+ (self.team.id, recent_busy.id),
+ (self.team.id, old_quiet.id),
+ (self.team.id, recent_quiet.id),
+ ]
+
+ @parameterized.expand([("the whole tick", 1.0, 2, 1), ("half the tick", 0.5, 1, 1)])
+ def test_paying_pairs_and_overdue_free_pairs_come_first_and_one_org_takes_at_most_its_share(
+ self, _name: str, paying_budget_share: float, paying_picked: int, fresh_picked: int
+ ) -> None:
+ self.organization.has_active_subscription = True
+ self.organization.save()
+ free_org = Organization.objects.create(name="Free")
+ free_team = Team.objects.create(organization=free_org, name="Free")
+ paying_users = [self._user(f"paying{index}@example.com") for index in range(3)]
+ overdue, *fresh = (
+ User.objects.create_and_join(free_org, f"{name}@example.com", None)
+ for name in ("overdue", "free1", "free2")
)
- self._summary(
- busy, generated_at=NOW - datetime.timedelta(days=1), input_watermark=NOW - datetime.timedelta(days=3)
+ for user in paying_users:
+ self._summary(user, generated_at=NOW - datetime.timedelta(hours=20))
+ self._changes(user, 3, days_ago=0.1)
+ for user, summarized, due in (
+ (overdue, NOW - datetime.timedelta(days=20), NOW - datetime.timedelta(hours=10)),
+ *((user, NOW - datetime.timedelta(days=2), None) for user in fresh),
+ ):
+ UserActivitySummary.objects.for_team(free_team.id).create(
+ team_id=free_team.id,
+ user_id=user.id,
+ status=UserActivitySummary.Status.OK,
+ generated_at=summarized,
+ next_eligible_at=due,
+ )
+ self._changes(user, 3, days_ago=0.1, team=free_team)
+ config = ActivitySummaryConfig(
+ enabled=True, enroll_all_approved=True, max_org_share=0.5, paying_budget_share=paying_budget_share
)
- self._changes(stale, 1)
- self._changes(fresh, 1)
- # Rows at or before the watermark are not new.
- self._changes(busy, 2, days_ago=4)
- self._changes(busy, 2)
- self._changes(cold, 1)
- tasks = [UserTaskTime(team_id=self.team.id, user_id=busy.id, created_at=NOW - datetime.timedelta(hours=1))]
-
- assert self._select(self._config(min_units=1), tasks=tasks) == [
- (self.team.id, cold.id),
- (self.team.id, busy.id),
- (self.team.id, stale.id),
- (self.team.id, fresh.id),
+ self._seed(config)
+
+ selection = select_pairs(config, NOW, 4)
+
+ assert [(pair.team_id, pair.user_id) for pair in selection.pairs] == [
+ (free_team.id, overdue.id),
+ *((self.team.id, user.id) for user in paying_users[:paying_picked]),
+ *((free_team.id, user.id) for user in fresh[:fresh_picked]),
]
+ assert selection.due_pairs == 6
+ assert [
+ (tier.tier, tier.due_pairs, round(tier.oldest_due_hours or 0, 3), round(tier.fresh_share or 0, 3))
+ for tier in selection.tiers
+ ] == [("paying", 3, 0.0, 1.0), ("free", 3, 10.0, round(2 / 3, 3))]
@parameterized.expand([("every stale pair fits the share", 0.3, 2), ("the share caps the rebuilds", 0.1, 1)])
def test_a_stale_summary_is_due_without_new_rows_and_ranks_with_cold_starts(
@@ -246,7 +294,9 @@ class TestActivitySummaryQueue(BaseTest):
self._changes(user, 1, days_ago=5)
self._changes(busy, 3)
- config = self._config(rebuild_before=NOW - datetime.timedelta(days=2, hours=12), rebuild_budget_share=share)
+ config = self._config(
+ rebuild_before=NOW - datetime.timedelta(days=2, hours=12), rebuild_budget_share=share, max_org_share=1
+ )
assert self._select(config) == [
*[(self.team.id, before_rebuild.id), (self.team.id, old_version.id)][:rebuilds],
(self.team.id, busy.id),
@@ -312,6 +362,7 @@ class TestActivitySummaryQueue(BaseTest):
("the tick cap", {"max_pairs_per_tick": 1}, "phs_test", STOP_MAX_PAIRS_PER_TICK, 1),
("the daily pair cap", {"max_pairs_per_day": 1}, "phs_test", STOP_MAX_PAIRS_PER_DAY, 0),
("the daily cost cap", {"max_cost_usd_per_day": 0.1}, "phs_test", STOP_MAX_COST_USD_PER_DAY, 0),
+ ("the org share", {"max_pairs_per_tick": 2, "max_org_share": 0.5}, "phs_test", STOP_MAX_ORG_SHARE, 1),
("no cap", {}, "phs_test", None, 2),
("no gateway", {}, "", STOP_GATEWAY_NOT_CONFIGURED, 0),
]
@@ -521,13 +572,15 @@ def _plan(pairs: int, **overrides: Any) -> ActivitySummaryTickPlan:
"emit_embeddings": True,
"capture_events": True,
"limits": DEFAULT_LIMITS,
+ "max_error_rate": 0.5,
+ "tiers": [ActivitySummaryTierStats(tier="paying", due_pairs=pairs)],
"stop_reason": None,
}
return ActivitySummaryTickPlan(**{**fields, **overrides})
async def _run_tick(
- plan: ActivitySummaryTickPlan, pair_cost: float | None
+ plan: ActivitySummaryTickPlan, pair_cost: float | None, pair_status: str = "ok"
) -> tuple[ActivitySummaryTickResult, int, list[ActivitySummaryTickResult]]:
running = peak = 0
captured: list[ActivitySummaryTickResult] = []
@@ -543,7 +596,7 @@ async def _run_tick(
peak = max(peak, running)
await asyncio.sleep(0.01)
running -= 1
- return ActivitySummaryPairResult(status="ok", cost_usd=pair_cost, called_model=True)
+ return ActivitySummaryPairResult(status=pair_status, cost_usd=pair_cost, called_model=True)
@activity.defn(name="capture_activity_summary_tick_activity")
async def fake_capture(result: ActivitySummaryTickResult) -> None:
@@ -568,21 +621,24 @@ async def _run_tick(
@pytest.mark.asyncio
@pytest.mark.parametrize(
- "plan,pair_cost,picked,stop_reason",
+ "plan,pair_cost,pair_status,picked,stop_reason",
[
- (_plan(5), 0.01, 5, None),
- (_plan(5, updates_left=3), 0.01, 3, STOP_MAX_PAIRS_PER_DAY),
+ (_plan(5), 0.01, "ok", 5, None),
+ (_plan(5, updates_left=3), 0.01, "ok", 3, STOP_MAX_PAIRS_PER_DAY),
# A missing cost counts at the bound, so the second wave spends the last of the budget.
- (_plan(6, cost_left_usd=3.0), None, 4, STOP_MAX_COST_USD_PER_DAY),
+ (_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY),
+ # The breaker waits for a sample of 10 model calls, which is five waves.
+ (_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE),
],
)
-async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps(
- plan: ActivitySummaryTickPlan, pair_cost: float | None, picked: int, stop_reason: str | None
+async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps_and_the_error_rate(
+ plan: ActivitySummaryTickPlan, pair_cost: float | None, pair_status: str, picked: int, stop_reason: str | None
) -> None:
- result, peak, captured = await _run_tick(plan, pair_cost)
+ result, peak, captured = await _run_tick(plan, pair_cost, pair_status)
assert peak <= plan.max_concurrency
- assert (result.picked, result.statuses, result.stop_reason) == (picked, {"ok": picked}, stop_reason)
+ assert (result.picked, result.statuses, result.stop_reason) == (picked, {pair_status: picked}, stop_reason)
+ assert result.tiers == plan.tiers
assert captured == [result]
diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.py
index a94e8f9f473c..67ff9e3618fd 100644
--- a/products/signals/backend/test/test_activity_summary_update.py
+++ b/products/signals/backend/test/test_activity_summary_update.py
@@ -271,8 +271,13 @@ class TestUpdateActivitySummary(BaseTest):
assert skipped is not None
assert (skipped.status, skipped.summary) == (UserActivitySummary.Status.SKIPPED, None)
assert skipped.next_eligible_at == NOW + datetime.timedelta(hours=6)
-
with time_machine.travel(NOW + datetime.timedelta(hours=7), tick=False):
+ assert self._update(llm).status == UpdateStatus.SKIPPED_TOO_FEW_UNITS
+ skipped_again = self._stored()
+ assert skipped_again is not None
+ assert skipped_again.next_eligible_at == NOW + datetime.timedelta(hours=7 + 12)
+
+ with time_machine.travel(NOW + datetime.timedelta(hours=20), tick=False):
assert self._update(FakeLLM([ANSWER]), min_units=2).status == UpdateStatus.OK
before = self._stored()
assert before is not None