:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): select activity summary pairs from a cursor-fed queue (#121687)
posthog[bot]078bbb409b03
diff --git a/.github/scripts/check-idor-model-coverage.py b/.github/scripts/check-idor-model-coverage.pyindex 9cd6bfac65e5..ffe2c08f1681 100644--- a/.github/scripts/check-idor-model-coverage.py+++ b/.github/scripts/check-idor-model-coverage.py@@ -364,6 +364,7 @@ def get_scoped_models() -> tuple[dict[str, set[str]], set[str], set[str], set[st"EventIngestionRestrictionConfig","GlobalRateLimitThresholdConfig","MessagingRecord",+ "UserActivitySummaryQueueCursor", # one global read position per input source, no rows of a team}# Baseline violations — these models SHOULD have team_id but don't yet.diff --git a/.semgrep/rules/security/idor-team-scoped-models.yaml b/.semgrep/rules/security/idor-team-scoped-models.yamlindex 8f2fc843ba99..03b3bfade471 100644--- a/.semgrep/rules/security/idor-team-scoped-models.yaml+++ b/.semgrep/rules/security/idor-team-scoped-models.yaml@@ -462,6 +462,7 @@ rules:|TracingView|UploadedMedia|UserInterview+ |UserActivitySummaryQueueState|UserActivitySummaryVersion|UserInterviewTopic|UserRepoPreference@@ -930,6 +931,7 @@ rules:|TracingView|UploadedMedia|UserInterview+ |UserActivitySummaryQueueState|UserActivitySummaryVersion|UserInterviewTopic|UserRepoPreferencediff --git a/products/signals/backend/activity_summary/config.py b/products/signals/backend/activity_summary/config.pyindex 0003aff3a618..bf3b7a301405 100644--- a/products/signals/backend/activity_summary/config.py+++ b/products/signals/backend/activity_summary/config.py@@ -28,6 +28,8 @@ ACTIVITY_SUMMARY_FLAG_DISTINCT_ID = "signals_activity_summary"@frozenclass ActivitySummaryConfig:enabled: bool = False+ # Enrolls every organization with AI data processing approval. Otherwise only `organization_ids`.+ enroll_all_approved: bool = Falseorganization_ids: tuple[str, ...] = ()skip_team_ids: tuple[int, ...] = ()max_pairs_per_tick: int = 50@@ -40,9 +42,13 @@ class ActivitySummaryConfig:max_versions: int = MAX_VERSIONSemit_embeddings: bool = Truecapture_events: bool = True+ # Count viewed objects in the queue. Turn it on only when the update reads views as an input.+ count_views: bool = Falselimits: RenderLimits = DEFAULT_LIMITS# Summaries generated before this time are rebuilt from the full window, as after a version change.rebuild_before: datetime.datetime | None = None+ # The largest share of a tick that rebuilds of stale summaries can take. Other updates get the rest.+ rebuild_budget_share: float = 0.3DISABLED = ActivitySummaryConfig()@@ -59,6 +65,11 @@ def _positive_number(value: Any, default: float) -> float:return float(value)+def _fraction(value: Any, default: float) -> float:+ number = _positive_number(value, default)+ return number if number <= 1 else default++def _bool(value: Any, default: bool) -> bool:return value if isinstance(value, bool) else default@@ -106,6 +117,7 @@ def parse_config(payload: Any) -> ActivitySummaryConfig:defaults = ActivitySummaryConfig()return ActivitySummaryConfig(enabled=True,+ enroll_all_approved=_bool(payload.get("enroll_all_approved"), defaults.enroll_all_approved),organization_ids=_organization_ids(payload.get("organization_ids")),skip_team_ids=_team_ids(payload.get("skip_team_ids")),max_pairs_per_tick=_positive_int(payload.get("max_pairs_per_tick"), defaults.max_pairs_per_tick),@@ -119,8 +131,10 @@ def parse_config(payload: Any) -> ActivitySummaryConfig:max_versions=_positive_int(payload.get("max_versions"), defaults.max_versions),emit_embeddings=_bool(payload.get("emit_embeddings"), defaults.emit_embeddings),capture_events=_bool(payload.get("capture_events"), defaults.capture_events),+ count_views=_bool(payload.get("count_views"), defaults.count_views),limits=parse_limits(payload.get("limits")),rebuild_before=_timestamp(payload.get("rebuild_before")),+ rebuild_budget_share=_fraction(payload.get("rebuild_budget_share"), defaults.rebuild_budget_share),)@@ -129,6 +143,7 @@ def read_config() -> ActivitySummaryConfig:logger.info("activity_summary_config_read",enabled=config.enabled,+ enroll_all_approved=config.enroll_all_approved,organizations=len(config.organization_ids),)return configdiff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.pyindex 042fce58c7a7..a1d2752e9bd5 100644--- a/products/signals/backend/activity_summary/queue.py+++ b/products/signals/backend/activity_summary/queue.py@@ -1,31 +1,43 @@"""Pick the (team, user) pairs that the next activity summary tick updates, and measure the daily caps.-Row counts here are only a queue estimate. `update_activity_summary` still applies the real drop-rules and `min_units`, and backs a pair off when it skips, so the pair leaves the queue head.+Each pair has a queue state row. One cursor per input source reads the new rows of all enrolled teams+and adds them to the pending units of their pair. `select_pairs` then reads the due pairs in one+ordered query. Row counts here are only a queue estimate. `update_activity_summary` still applies the+real drop rules and `min_units`, and backs a pair off when it skips, so the pair leaves the queue head."""+import mathimport datetime-from collections import Counter+from collections.abc import Callable+from typing import Any-from django.db.models import Count, OuterRef, Q, Subquery, Sum-from django.db.models.functions import Coalesce+from django.db import transaction+from django.db.models import BooleanField, Case, Count, ExpressionWrapper, F, Max, OuterRef, Q, Subquery, Sum, When+from django.db.models.functions import Coalesce, Greatestfrom posthog.dataclasses import frozen-from posthog.models import ActivityLog, Team+from posthog.models import ActivityLog, FileSystemViewLog, Teamfrom products.signals.backend.activity_summary.config import ActivitySummaryConfigfrom products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS-from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX, is_stale-from products.signals.backend.activity_summary.prompt import system_prompt+from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX+from products.signals.backend.activity_summary.prompt import PROMPT_VERSION, system_promptfrom products.signals.backend.activity_summary.render import (DEFAULT_LIMITS,MAX_LIMITS,MAX_SUMMARY_CHARS,+ NOISY_ACTIVITY_SCOPES,+ RENDER_VERSION,+ SANDBOX_PROMPT_TITLE_PREFIX,SCOUT_CLIENT_PREFIX,RenderLimits,)from products.signals.backend.activity_summary.summaries import DEFAULT_MODEL, MAX_ATTEMPTS, MODEL_PRICES, cost_usd-from products.signals.backend.models import UserActivitySummary+from products.signals.backend.models import (+ UserActivitySummary,+ UserActivitySummaryQueueCursor,+ UserActivitySummaryQueueState,+)from products.tasks.backend.facade.user_task_summaries import list_user_task_timesDAY = datetime.timedelta(days=1)@@ -35,6 +47,22 @@ _BOUND_CHARS_PER_TOKEN = 2_UNPRICED_UPDATE_COST_USD = 0.05_CALLED_MODEL = (UserActivitySummary.Status.OK, UserActivitySummary.Status.FAILED)+SOURCE_ACTIVITY_LOG = "activity_log"+SOURCE_TASKS = "tasks"+SOURCE_VIEWS = "views"+# A cursor stays this far behind now, so it still reads a row that commits a little after its timestamp.+FEED_LAG = datetime.timedelta(minutes=5)+# One tick reads at most this much of one source. A cursor that fell behind catches up over the next ticks.+MAX_FEED_STEP = datetime.timedelta(hours=6)+# Pairs with no summary first, then the most pending units, then the oldest summary. The state index has this order.+_QUEUE_ORDER = (+ ExpressionWrapper(Q(summarized_at__isnull=False), output_field=BooleanField()).asc(),+ F("pending_units").desc(),+ F("summarized_at").asc(),+ "team_id",+ "user_id",+)+@frozen(order=True)class ActivitySummaryPair:@@ -43,18 +71,13 @@ class ActivitySummaryPair:@frozen-class _Stored:- input_watermark: datetime.datetime | None- generated_at: datetime.datetime | None- next_eligible_at: datetime.datetime | None- prompt_version: int | None- render_version: int | None+class _NewRows:+ units: int+ last_at: datetime.datetime-@frozen-class _RowCounts:- new: int = 0- window: int = 0+_Pending = dict[ActivitySummaryPair, _NewRows]+_Reader = Callable[[list[int], datetime.datetime, datetime.datetime, bool], _Pending]@frozen@@ -105,112 +128,285 @@ def daily_usage(now: datetime.datetime) -> DailyUsage:)-def _enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:- if not config.organization_ids:- return []- return list(- Team.objects.filter(- organization_id__in=config.organization_ids,- organization__is_ai_data_processing_approved=True,- )- .exclude(id__in=config.skip_team_ids)- .values_list("id", flat=True)- )+def enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:+ """The teams of every approved organization, or of the approved organizations in the allowlist."""+ teams = Team.objects.filter(organization__is_ai_data_processing_approved=True)+ if not config.enroll_all_approved:+ if not config.organization_ids:+ return []+ teams = teams.filter(organization_id__in=config.organization_ids)+ return list(teams.exclude(id__in=config.skip_team_ids).values_list("id", flat=True))++def _add(pending: _Pending, pair: ActivitySummaryPair, units: int, at: datetime.datetime) -> None:+ current = pending.get(pair)+ if current is None:+ pending[pair] = _NewRows(units=units, last_at=at)+ else:+ pending[pair] = _NewRows(units=current.units + units, last_at=max(current.last_at, at))-def _stored_rows(team_ids: list[int]) -> dict[ActivitySummaryPair, _Stored]:++def _watermarks(team_ids: list[int]) -> dict[ActivitySummaryPair, datetime.datetime | None]:# nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)rows = UserActivitySummary.all_teams.filter(team_id__in=team_ids).values_list(- "team_id", "user_id", "input_watermark", "generated_at", "next_eligible_at", "prompt_version", "render_version"+ "team_id", "user_id", "input_watermark")- return {- ActivitySummaryPair(team_id=team_id, user_id=user_id): _Stored(- input_watermark=watermark,- generated_at=generated_at,- next_eligible_at=eligible,- prompt_version=prompt_version,- render_version=render_version,- )- for team_id, user_id, watermark, generated_at, eligible, prompt_version, render_version in rows- }+ return {ActivitySummaryPair(team_id=team_id, user_id=user_id): watermark for team_id, user_id, watermark in rows}+++def _after_watermark(+ rows: list[tuple[int, int, datetime.datetime]], team_ids: list[int], after_watermark: bool+) -> _Pending:+ watermarks = _watermarks(team_ids) if after_watermark else {}+ pending: _Pending = {}+ for team_id, user_id, at in rows:+ pair = ActivitySummaryPair(team_id=team_id, user_id=user_id)+ watermark = watermarks.get(pair)+ # A row the summary already read is no pending unit, but it still dates the pair's last activity.+ _add(pending, pair, 1 if watermark is None or at > watermark else 0, at)+ return pendingdef _activity_rows(- team_ids: list[int], window_start: datetime.datetime, now: datetime.datetime-) -> dict[ActivitySummaryPair, _RowCounts]:- """Activity log rows per pair after the pair's watermark and in the window, with the scope filters of `inputs.py`."""- watermark = Subquery(- # nosemgrep: idor-lookup-without-team (correlated on the outer row's team_id)- UserActivitySummary.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id")).values(- "input_watermark"- )[:1]- )+ team_ids: list[int], start: datetime.datetime, end: datetime.datetime, after_watermark: bool+) -> _Pending:+ """Activity log rows per pair in [start, end), with the filters of `inputs.user_activity`."""rows = (ActivityLog.objects.filter(- team_id__in=team_ids, user_id__isnull=False, created_at__gte=window_start, created_at__lt=now+ team_id__in=team_ids, user_id__isnull=False, created_at__gte=start, created_at__lt=end).exclude(is_system=True).exclude(was_impersonated=True)+ .exclude(scope__in=NOISY_ACTIVITY_SCOPES).exclude(client__startswith=SCOUT_CLIENT_PREFIX)- .values("team_id", "user_id")- .annotate(new=Count("id", filter=Q(created_at__gt=Coalesce(watermark, window_start))), window=Count("id"))- .values_list("team_id", "user_id", "new", "window"))+ unit_count = Count("id")+ if after_watermark:+ watermark = Subquery(+ # nosemgrep: idor-lookup-without-team (correlated on the outer row's team_id)+ UserActivitySummary.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id")).values(+ "input_watermark"+ )[:1]+ )+ # A row the summary already read is no pending unit, but it still dates the pair's last activity.+ unit_count = Count("id", filter=Q(created_at__gt=Coalesce(watermark, start)))return {- ActivitySummaryPair(team_id=team_id, user_id=user_id): _RowCounts(new=new, window=window)- for team_id, user_id, new, window in rows+ ActivitySummaryPair(team_id=team_id, user_id=user_id): _NewRows(units=units, last_at=last_at)+ for team_id, user_id, units, last_at in rows.values("team_id", "user_id")+ .annotate(units=unit_count, last_at=Max("created_at"))+ .values_list("team_id", "user_id", "units", "last_at")if team_id is not None and user_id is not None}def _task_rows(- team_ids: list[int],- window_start: datetime.datetime,- now: datetime.datetime,- stored: dict[ActivitySummaryPair, _Stored],-) -> dict[ActivitySummaryPair, _RowCounts]:- """Tasks per pair after the pair's watermark and in the window."""- new: Counter[ActivitySummaryPair] = Counter()- window: Counter[ActivitySummaryPair] = Counter()- for task in list_user_task_times(- team_ids=team_ids, start=window_start, end=now, origin_products=TASK_ORIGIN_PRODUCTS- ):- pair = ActivitySummaryPair(team_id=task.team_id, user_id=task.user_id)- window[pair] += 1- row = stored.get(pair)- if row is None or row.input_watermark is None or task.created_at > row.input_watermark:- new[pair] += 1- return {pair: _RowCounts(new=new[pair], window=count) for pair, count in window.items()}+ team_ids: list[int], start: datetime.datetime, end: datetime.datetime, after_watermark: bool+) -> _Pending:+ """Tasks per pair in [start, end), with the filters of `inputs.user_tasks`."""+ tasks = list_user_task_times(+ team_ids=team_ids,+ start=start,+ end=end,+ origin_products=TASK_ORIGIN_PRODUCTS,+ exclude_title_prefixes=(SANDBOX_PROMPT_TITLE_PREFIX,),+ )+ return _after_watermark(+ [(task.team_id, task.user_id, task.created_at) for task in tasks], team_ids, after_watermark+ )+++def _view_rows(+ team_ids: list[int], start: datetime.datetime, end: datetime.datetime, after_watermark: bool+) -> _Pending:+ """Viewed objects and scenes per pair with a latest view in [start, end). Folders are not views."""+ if not team_ids:+ return {}+ rows = (+ FileSystemViewLog.objects.filter(team_id__in=team_ids, viewed_at__gte=start, viewed_at__lt=end)+ .exclude(type="folder")+ .values_list("team_id", "user_id", "viewed_at")+ )+ return _after_watermark(list(rows), team_ids, after_watermark)+++_READERS: dict[str, _Reader] = {+ SOURCE_ACTIVITY_LOG: _activity_rows,+ SOURCE_TASKS: _task_rows,+ SOURCE_VIEWS: _view_rows,+}+++def queue_sources(config: ActivitySummaryConfig) -> tuple[str, ...]:+ return (+ (SOURCE_ACTIVITY_LOG, SOURCE_TASKS, SOURCE_VIEWS) if config.count_views else (SOURCE_ACTIVITY_LOG, SOURCE_TASKS)+ )+++_STORED_FIELDS = ("next_eligible_at", "generated_at", "prompt_version", "render_version")+++def _state_fields(stored: dict[str, Any]) -> dict[str, Any]:+ """The queue state fields that copy a stored summary."""+ return {+ "due_at": stored["next_eligible_at"],+ "summarized_at": stored["generated_at"],+ "prompt_version": stored["prompt_version"],+ "render_version": stored["render_version"],+ }+++def _create_missing_states(pairs: list[ActivitySummaryPair]) -> None:+ """Create a state row for each pair that has none, with the times and versions of its stored summary."""+ team_ids = {pair.team_id for pair in pairs}+ # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the given pairs)+ stored = UserActivitySummary.all_teams.filter(+ team_id__in=team_ids, user_id__in={pair.user_id for pair in pairs}+ ).values("team_id", "user_id", *_STORED_FIELDS)+ copies = {ActivitySummaryPair(team_id=row["team_id"], user_id=row["user_id"]): _state_fields(row) for row in stored}+ UserActivitySummaryQueueState.all_teams.bulk_create(+ [+ UserActivitySummaryQueueState(team_id=pair.team_id, user_id=pair.user_id, **copies.get(pair, {}))+ for pair in pairs+ ],+ ignore_conflicts=True,+ )+++def _add_pending(pending: _Pending) -> None:+ if not pending:+ return+ _create_missing_states(list(pending))+ for pair, rows in pending.items():+ UserActivitySummaryQueueState.objects.for_team(pair.team_id).filter(user_id=pair.user_id).update(+ pending_units=F("pending_units") + rows.units,+ # Postgres GREATEST skips a null.+ last_activity_at=Greatest("last_activity_at", rows.last_at),+ )+++def feed_queue(config: ActivitySummaryConfig, now: datetime.datetime) -> int:+ """Move each source cursor forward and add the rows it passes to the queue state. Returns the pairs fed.++ A source with no cursor starts at the current end, so the backfill command seeds the earlier rows.+ """+ team_ids = enrolled_team_ids(config)+ end = now - FEED_LAG+ fed = 0+ for source in queue_sources(config):+ with transaction.atomic():+ cursor = UserActivitySummaryQueueCursor.objects.select_for_update().filter(source=source).first()+ if cursor is None:+ UserActivitySummaryQueueCursor.objects.get_or_create(source=source, defaults={"position": end})+ continue+ stop = min(end, cursor.position + MAX_FEED_STEP)+ if stop <= cursor.position:+ continue+ pending = _READERS[source](team_ids, cursor.position, stop, False)+ _add_pending(pending)+ cursor.position = stop+ cursor.save(update_fields=["position", "updated_at"])+ fed += len(pending)+ return fed+++def seed_queue(config: ActivitySummaryConfig, team_ids: list[int], end: datetime.datetime) -> int:+ """Set the queue state of these teams from the rows in the window before `end`.++ Pending units count the rows after each pair's watermark. Last activity counts every row, so a+ stale summary with no new rows is still due.++ Pending units of these teams that the window does not hold go to zero. Returns the pairs seeded.+ """+ if not team_ids:+ return 0+ window_start = end - datetime.timedelta(days=config.limits.window_days)+ pending: _Pending = {}+ for source in queue_sources(config):+ for pair, rows in _READERS[source](team_ids, window_start, end, True).items():+ _add(pending, pair, rows.units, rows.last_at)+ with transaction.atomic():+ # nosemgrep: idor-lookup-without-team (operator backfill with no user input; filtered to the given teams)+ UserActivitySummaryQueueState.all_teams.filter(team_id__in=team_ids).update(pending_units=0)+ _add_pending(pending)+ return len(pending)+++def move_cursors_to(config: ActivitySummaryConfig, end: datetime.datetime) -> None:+ """Move each cursor to `end` unless it is already past it, so the feed does not count seeded rows again."""+ for source in queue_sources(config):+ cursor, created = UserActivitySummaryQueueCursor.objects.get_or_create(+ source=source, defaults={"position": end}+ )+ if not created and cursor.position < end:+ cursor.position = end+ cursor.save(update_fields=["position", "updated_at"])+++def sync_pair_state(team_id: int, user_id: int) -> None:+ """Copy the times and versions of the stored summary to the pair's queue state.++ Pending units go to zero when the summary's watermark includes the newest pending row.+ """+ stored = (+ UserActivitySummary.objects.for_team(team_id)+ .filter(user_id=user_id)+ .values(*_STORED_FIELDS, "input_watermark")+ .first()+ )+ if stored is None:+ return+ watermark = stored["input_watermark"]+ UserActivitySummaryQueueState.objects.for_team(team_id).filter(user_id=user_id).update(+ **_state_fields(stored),+ pending_units=F("pending_units")+ if watermark is None+ else Case(When(last_activity_at__lte=watermark, then=0), default=F("pending_units")),+ )+++def _current(config: ActivitySummaryConfig) -> Q:+ """Queue states with no summary, or a summary that `logic.is_stale` does not mark stale."""+ current = Q(prompt_version=PROMPT_VERSION, render_version=RENDER_VERSION)+ if config.rebuild_before is not None:+ current &= Q(summarized_at__gte=config.rebuild_before)+ return Q(summarized_at__isnull=True) | currentdef select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: int) -> list[ActivitySummaryPair]:"""The due pairs of the enrolled teams, highest priority first, cut to `limit`.- A pair is due when it has a task or an activity log row after its watermark, or in the last- `window_days` with no watermark or a stale summary, and its `next_eligible_at` is empty or past.- Pairs with no summary or a stale summary come first, then the pairs with the most new rows,- then the oldest summaries.+ A pair is due when it has a row in the last `window_days` and its due time is empty or past. A pair+ with a current summary also needs at least `min_units` pending units. One unit never comes from less+ than one row, so a pair with fewer rows would only skip. A pair with a stale summary needs no pending+ units, because its update reads the full window. Stale pairs take at most `rebuild_budget_share` of+ `limit`, so a version change does not stop the other updates.++ Pairs with no summary come first, then stale pairs with the oldest summary first, then the pairs with+ the most pending units, then the oldest summaries."""- team_ids = _enrolled_team_ids(config)+ team_ids = enrolled_team_ids(config)if not team_ids or limit <= 0:return []window_start = now - datetime.timedelta(days=config.limits.window_days)- stored = _stored_rows(team_ids)- activity_rows = _activity_rows(team_ids, window_start, now)- task_rows = _task_rows(team_ids, window_start, now, stored)-- due: list[tuple[bool, int, datetime.datetime, ActivitySummaryPair]] = []- for pair in activity_rows.keys() | task_rows.keys():- row = stored.get(pair)- if row is not None and row.next_eligible_at is not None and row.next_eligible_at > now:- continue- activity = activity_rows.get(pair, _RowCounts())- tasks = task_rows.get(pair, _RowCounts())- stale = is_stale(row, config.rebuild_before)- count = activity.window + tasks.window if stale else activity.new + tasks.new- if not count:- continue- generated_at = row.generated_at if row is not None else None- due.append((generated_at is not None and not stale, -count, generated_at or window_start, pair))- due.sort()- return [pair for *_, pair in due[:limit]]+ due = (+ # nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)+ UserActivitySummaryQueueState.all_teams.filter(team_id__in=team_ids, last_activity_at__gte=window_start).filter(+ Q(due_at__isnull=True) | Q(due_at__lte=now)+ )+ )+ current = _current(config)+ updates = list(+ due.filter(current, pending_units__gte=max(1, config.min_units))+ .order_by(*_QUEUE_ORDER)+ .values_list("team_id", "user_id", "summarized_at")[:limit]+ )+ rebuilds = list(+ due.exclude(current)+ .order_by("summarized_at", "team_id", "user_id")+ .values_list("team_id", "user_id")[: math.ceil(limit * config.rebuild_budget_share)]+ )+ cold = [(team_id, user_id) for team_id, user_id, at in updates if at is None]+ warm = [(team_id, user_id) for team_id, user_id, at in updates if at is not None]+ return [ActivitySummaryPair(team_id=team_id, user_id=user_id) for team_id, user_id in cold + rebuilds + warm][+ :limit+ ]diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.pyindex 07b72a61e97b..4a535621749e 100644--- a/products/signals/backend/activity_summary/tick.py+++ b/products/signals/backend/activity_summary/tick.py@@ -38,8 +38,10 @@ from products.signals.backend.activity_summary.prompt import PROMPT_VERSIONfrom products.signals.backend.activity_summary.queue import (ActivitySummaryPair,daily_usage,+ feed_queue,max_update_cost_usd,select_pairs,+ sync_pair_state,)from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, RENDER_VERSION, RenderLimitsfrom products.signals.backend.activity_summary.summaries import DEFAULT_MODEL, PIPELINE_DISTINCT_ID@@ -133,6 +135,9 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP# Every pair would fail on the same missing gateway, so the tick runs none.logger.warning("activity_summary_tick_gateway_not_configured")enabled = config.enabled and not gateway_missing+ if config.enabled:+ # The cursors move even when a cap stops the tick, so new rows do not pile up behind them.+ feed_queue(config, now)usage = daily_usage(now) if enabled else Noneupdates_left = max(0, config.max_pairs_per_day - usage.updates) if usage else 0cost_left = max(0.0, config.max_cost_usd_per_day - usage.cost_usd) if usage else 0.0@@ -239,6 +244,11 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A# Without a failed row and a backoff, the queue picks the same pair again on the next tick.update = record_update_exception(input.team_id, input.user_id, error)called_model = isinstance(error, UpdateAfterModelCallError)+ try:+ sync_pair_state(input.team_id, input.user_id)+ except Exception:+ # The pair stays due, so the next tick picks it, skips it as not eligible yet, and syncs it then.+ logger.exception("activity_summary_queue_sync_failed", team_id=input.team_id, user_id=input.user_id)if update.status == UpdateStatus.OK and input.emit_embeddings:try:_emit_embedding(input.team_id, input.user_id)diff --git a/products/signals/backend/management/commands/backfill_activity_summary_queue.py b/products/signals/backend/management/commands/backfill_activity_summary_queue.pynew file mode 100644index 000000000000..882aec17a1fe--- /dev/null+++ b/products/signals/backend/management/commands/backfill_activity_summary_queue.py@@ -0,0 +1,43 @@+"""Seed the activity summary queue state of the enrolled teams, in batches of teams.++Run it once before the tick reads the queue, and again after a large enrollment change. Without it,+the queue holds only the rows that the cursors read after their first tick.+"""++import json++from django.core.management.base import BaseCommand+from django.utils import timezone++from products.signals.backend.activity_summary.config import parse_config, read_config+from products.signals.backend.activity_summary.queue import FEED_LAG, enrolled_team_ids, move_cursors_to, seed_queue+++class Command(BaseCommand):+ help = "Seed the activity summary queue state from the rows after each pair's watermark, then move the cursors."++ def add_arguments(self, parser):+ parser.add_argument("--batch-size", type=int, default=50, help="Teams per batch.")+ parser.add_argument(+ "--config",+ type=json.loads,+ default=None,+ help="A flag payload as JSON, in place of the `activity-summary` flag, e.g. "+ '\'{"enabled": true, "enroll_all_approved": true}\'.',+ )++ def handle(self, *args, **options):+ config = parse_config(options["config"]) if options["config"] is not None else read_config()+ if not config.enabled:+ self.stdout.write("The config is not enabled, so no team is enrolled.")+ return+ batch_size = max(1, options["batch_size"])+ end = timezone.now() - FEED_LAG+ team_ids = enrolled_team_ids(config)+ pairs = 0+ for index in range(0, len(team_ids), batch_size):+ batch = team_ids[index : index + batch_size]+ pairs += seed_queue(config, batch, end)+ self.stdout.write(f"seeded {index + len(batch)}/{len(team_ids)} teams, {pairs} pairs")+ move_cursors_to(config, end)+ self.stdout.write(f"done: {len(team_ids)} teams, {pairs} pairs, cursors at {end.isoformat()}")diff --git a/products/signals/backend/migrations/0192_useractivitysummary_queue.py b/products/signals/backend/migrations/0192_useractivitysummary_queue.pynew file mode 100644index 000000000000..92da85026f05--- /dev/null+++ b/products/signals/backend/migrations/0192_useractivitysummary_queue.py@@ -0,0 +1,91 @@+# Generated by Django 5.2.18 on 2026-10-21 21:43++import django.db.models.manager+import django.db.models.deletion+from django.conf import settings+from django.db import migrations, models++import posthog.uuidt+++class Migration(migrations.Migration):+ dependencies = [+ ("posthog", "1407_scimprovisioneduser_scimrequestlog"),+ ("signals", "0191_useractivitysummaryversion"),+ migrations.swappable_dependency(settings.AUTH_USER_MODEL),+ ]++ operations = [+ migrations.CreateModel(+ name="UserActivitySummaryQueueCursor",+ fields=[+ (+ "source",+ models.CharField(max_length=40, primary_key=True, serialize=False),+ ),+ ("position", models.DateTimeField()),+ ("updated_at", models.DateTimeField(auto_now=True)),+ ],+ ),+ migrations.CreateModel(+ name="UserActivitySummaryQueueState",+ fields=[+ (+ "id",+ models.UUIDField(+ default=posthog.uuidt.uuid7,+ editable=False,+ primary_key=True,+ serialize=False,+ ),+ ),+ ("pending_units", models.IntegerField(default=0)),+ ("last_activity_at", models.DateTimeField(blank=True, null=True)),+ ("due_at", models.DateTimeField(blank=True, null=True)),+ ("summarized_at", models.DateTimeField(blank=True, null=True)),+ ("prompt_version", models.PositiveSmallIntegerField(blank=True, null=True)),+ ("render_version", models.PositiveSmallIntegerField(blank=True, null=True)),+ ("created_at", models.DateTimeField(auto_now_add=True)),+ ("updated_at", models.DateTimeField(auto_now=True)),+ (+ "team",+ models.ForeignKey(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to="posthog.team",+ ),+ ),+ (+ "user",+ models.ForeignKey(+ db_constraint=False,+ on_delete=django.db.models.deletion.CASCADE,+ related_name="+",+ to=settings.AUTH_USER_MODEL,+ ),+ ),+ ],+ options={+ "default_manager_name": "all_teams",+ "indexes": [+ models.Index(+ models.ExpressionWrapper(+ models.Q(("summarized_at__isnull", False)),+ output_field=models.BooleanField(),+ ),+ models.OrderBy(models.F("pending_units"), descending=True),+ models.OrderBy(models.F("summarized_at")),+ condition=models.Q(("pending_units__gt", 0)),+ name="signals_activity_queue_order",+ )+ ],+ "constraints": [+ models.UniqueConstraint(fields=("team", "user"), name="signals_activity_queue_team_user")+ ],+ },+ managers=[+ ("all_teams", django.db.models.manager.Manager()),+ ],+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex 2eda61cec820..4699f622c187 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0191_useractivitysummaryversion+0192_useractivitysummary_queuediff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 2e61c8b2df55..dd31efc4f1ea 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -3912,3 +3912,55 @@ class UserActivitySummaryVersion(TeamScopedRootMixin, UUIDModel):constraints = [models.UniqueConstraint(fields=["summary_row", "version"], name="signals_activity_summary_version_unique"),]+++class UserActivitySummaryQueueState(TeamScopedRootMixin, UUIDModel):+ """The queue state of one (team, user) pair of the scheduled activity summary tick.++ The cursors of `activity_summary/queue.py` add new input rows to `pending_units`. The tick picks+ the due pairs in one ordered query and resets the state from the stored summary after an update.+ """++ # See SignalReportRefund.all_teams for rationale.+ all_teams = models.Manager() # noqa: DJ012++ team = models.ForeignKey("posthog.Team", on_delete=models.CASCADE, db_constraint=False, related_name="+")+ user = models.ForeignKey("posthog.User", on_delete=models.CASCADE, db_constraint=False, related_name="+")++ # New input rows since the last good update. An estimate: the update applies the real filters.+ pending_units = models.IntegerField(default=0)+ last_activity_at = models.DateTimeField(null=True, blank=True)+ # The pair is due when this is empty or past. A copy of `UserActivitySummary.next_eligible_at`.+ due_at = models.DateTimeField(null=True, blank=True)+ # A copy of `UserActivitySummary.generated_at`, so the queue order needs no join.+ summarized_at = models.DateTimeField(null=True, blank=True)+ # Copies of the stored summary's versions, so the queue finds stale summaries with no join.+ prompt_version = models.PositiveSmallIntegerField(null=True, blank=True)+ render_version = models.PositiveSmallIntegerField(null=True, blank=True)++ created_at = models.DateTimeField(auto_now_add=True)+ updated_at = models.DateTimeField(auto_now=True)++ class Meta:+ default_manager_name = "all_teams"+ constraints = [+ models.UniqueConstraint(fields=["team", "user"], name="signals_activity_queue_team_user"),+ ]+ indexes = [+ # The queue order of `select_pairs`: pairs with no summary, then most pending units, then oldest summary.+ models.Index(+ models.ExpressionWrapper(models.Q(summarized_at__isnull=False), output_field=models.BooleanField()),+ models.F("pending_units").desc(),+ models.F("summarized_at").asc(),+ name="signals_activity_queue_order",+ condition=models.Q(pending_units__gt=0),+ ),+ ]+++class UserActivitySummaryQueueCursor(models.Model):+ """The read position of one input source of the activity summary queue, over all teams."""++ source = models.CharField(max_length=40, primary_key=True)+ position = models.DateTimeField()+ updated_at = models.DateTimeField(auto_now=True)diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.pyindex 3d1b56e6bb93..9dac59470aeb 100644--- a/products/signals/backend/test/test_activity_summary_tick.py+++ b/products/signals/backend/test/test_activity_summary_tick.py@@ -19,7 +19,7 @@ from temporalio.converter import DataConverterfrom temporalio.testing import WorkflowEnvironmentfrom temporalio.worker import UnsandboxedWorkflowRunner, Worker-from posthog.models import ActivityLog, Organization, Team, User+from posthog.models import ActivityLog, FileSystemViewLog, Organization, Team, Userfrom products.signals.backend.activity_summary import (schedule as tick_schedule,@@ -31,7 +31,10 @@ from products.signals.backend.activity_summary.queue import (ActivitySummaryPair,DailyUsage,daily_usage,+ enrolled_team_ids,+ feed_queue,max_update_cost_usd,+ seed_queue,select_pairs,)from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, MAX_LIMITS, RENDER_VERSION, RenderLimits@@ -53,7 +56,7 @@ from products.signals.backend.activity_summary.tick import (plan_tick,run_pair,)-from products.signals.backend.models import UserActivitySummary+from products.signals.backend.models import UserActivitySummary, UserActivitySummaryQueueStatefrom products.signals.backend.test.test_activity_summary_render import FakeLLMfrom products.tasks.backend.facade.user_task_summaries import UserTaskTime@@ -90,6 +93,7 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = ["max_versions": -3,"emit_embeddings": "no","rebuild_before": "not a time",+ "rebuild_budget_share": 2,},ActivitySummaryConfig(enabled=True, organization_ids=("org-a",), skip_team_ids=(3,)),),@@ -103,6 +107,8 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = ["max_versions": 5,"capture_events": False,"rebuild_before": "2026-10-01T00:00:00",+ "enroll_all_approved": True,+ "count_views": True,},ActivitySummaryConfig(enabled=True,@@ -112,6 +118,8 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [max_versions=5,capture_events=False,rebuild_before=datetime.datetime(2026, 10, 1, tzinfo=datetime.UTC),+ enroll_all_approved=True,+ count_views=True,),),(@@ -176,9 +184,22 @@ class TestActivitySummaryQueue(BaseTest):**({"prompt_version": PROMPT_VERSION, "render_version": RENDER_VERSION} | fields),)- def _select(self, config: ActivitySummaryConfig | None = None, tasks: list[UserTaskTime] | None = None):+ def _seed(self, config: ActivitySummaryConfig | None = None, tasks: list[UserTaskTime] | None = None) -> None:+ config = config or self._config()with patch("products.signals.backend.activity_summary.queue.list_user_task_times", return_value=tasks or []):- return [(pair.team_id, pair.user_id) for pair in select_pairs(config or self._config(), NOW, 10)]+ seed_queue(config, enrolled_team_ids(config), NOW)++ def _select(+ self,+ config: ActivitySummaryConfig | None = None,+ tasks: list[UserTaskTime] | None = None,+ *,+ seed: bool = True,+ now: datetime.datetime = NOW,+ ):+ if seed:+ self._seed(config, tasks)+ return [(pair.team_id, pair.user_id) for pair in select_pairs(config or self._config(), now, 10)]def test_orders_cold_starts_then_most_new_rows_then_oldest_summary(self) -> None:stale, fresh, busy, cold = (self._user(f"{name}@example.com") for name in ("stale", "fresh", "busy", "cold"))@@ -199,14 +220,17 @@ class TestActivitySummaryQueue(BaseTest):self._changes(cold, 1)tasks = [UserTaskTime(team_id=self.team.id, user_id=busy.id, created_at=NOW - datetime.timedelta(hours=1))]- assert self._select(tasks=tasks) == [+ assert self._select(self._config(min_units=1), tasks=tasks) == [(self.team.id, cold.id),(self.team.id, busy.id),(self.team.id, stale.id),(self.team.id, fresh.id),]- def test_a_stale_summary_is_due_without_new_rows_and_ranks_with_cold_starts(self) -> None:+ @parameterized.expand([("every stale pair fits the share", 0.3, 2), ("the share caps the rebuilds", 0.1, 1)])+ def test_a_stale_summary_is_due_without_new_rows_and_ranks_with_cold_starts(+ self, _name: str, share: float, rebuilds: int+ ) -> None:current, old_version, before_rebuild, busy = (self._user(f"{name}@example.com") for name in ("current", "old", "rebuild", "busy"))@@ -222,10 +246,9 @@ class TestActivitySummaryQueue(BaseTest):self._changes(user, 1, days_ago=5)self._changes(busy, 3)- config = self._config(rebuild_before=NOW - datetime.timedelta(days=2, hours=12))+ config = self._config(rebuild_before=NOW - datetime.timedelta(days=2, hours=12), rebuild_budget_share=share)assert self._select(config) == [- (self.team.id, before_rebuild.id),- (self.team.id, old_version.id),+ *[(self.team.id, before_rebuild.id), (self.team.id, old_version.id)][:rebuilds],(self.team.id, busy.id),]@@ -246,6 +269,20 @@ class TestActivitySummaryQueue(BaseTest):assert self._select(self._config(skip_team_ids=(other_team.id,))) == [(self.team.id, due.id)]+ def test_enroll_all_approved_enrolls_every_approved_organization(self) -> None:+ user = self._user("[email protected]")+ approved = Team.objects.create(organization=Organization.objects.create(name="Approved"), name="Approved")+ not_approved = Team.objects.create(+ organization=Organization.objects.create(name="Not approved", is_ai_data_processing_approved=False),+ name="Not approved",+ )+ skipped = Team.objects.create(organization=approved.organization, name="Skipped")+ for team in (self.team, approved, not_approved, skipped):+ self._changes(user, 3, team=team)+ config = ActivitySummaryConfig(enabled=True, enroll_all_approved=True, skip_team_ids=(skipped.id,))++ assert sorted(self._select(config)) == sorted([(self.team.id, user.id), (approved.id, user.id)])+def test_selects_nothing_without_ai_data_processing_approval(self) -> None:self.organization.is_ai_data_processing_approved = Falseself.organization.save()@@ -285,6 +322,7 @@ class TestActivitySummaryQueue(BaseTest):self._changes(self._user("[email protected]"), 3)self._changes(self._user("[email protected]"), 3)self._summary(self.user, cost_usd=0.1)+ self._seed()with (override_settings(@@ -326,7 +364,9 @@ class TestActivitySummaryQueue(BaseTest):emit_embeddings=True,capture_events=True,)- assert self._select() == [(self.team.id, self.user.id)]+ # The queue picks a pair with fewer rows than the update's `min_units`, so the update skips it.+ config = self._config(min_units=1)+ assert self._select(config) == [(self.team.id, self.user.id)]with (patch("products.signals.backend.activity_summary.logic.gateway_client", **gateway),@@ -339,7 +379,7 @@ class TestActivitySummaryQueue(BaseTest):result = run_pair(pair_input)assert (result.status, result.called_model) == (status, called_model)- assert self._select() == []+ assert self._select(config, seed=False) == []if called_model:assert result.cost_usdusage = daily_usage(NOW)@@ -369,9 +409,57 @@ class TestActivitySummaryQueue(BaseTest):)assert (result.status, result.called_model) == ("failed", False)- assert self._select() == []+ assert self._select(seed=False) == []assert daily_usage(NOW) == DailyUsage(updates=1, cost_usd=0.25)+ @parameterized.expand([("counted", True, 1), ("not counted", False, 0)])+ def test_viewed_objects_make_a_pair_due_when_views_count(self, _name: str, count_views: bool, due: int) -> None:+ for index in range(3):+ FileSystemViewLog.objects.create(+ team=self.team,+ user=self.user,+ type="insight",+ ref=str(index),+ viewed_at=NOW - datetime.timedelta(hours=index + 1),+ )++ assert self._select(self._config(count_views=count_views)) == [(self.team.id, self.user.id)][:due]++ def test_the_feed_reads_each_row_once_and_an_ok_update_clears_the_pending_units(self) -> None:+ config = self._config()+ with patch("products.signals.backend.activity_summary.queue.list_user_task_times", return_value=[]):+ # The first feed only places the cursors. The backfill command seeds the earlier rows.+ feed_queue(config, NOW - datetime.timedelta(hours=1))+ self._changes(self.user, 3, days_ago=0.02)+ feed_queue(config, NOW)+ feed_queue(config, NOW)+ state = UserActivitySummaryQueueState.objects.for_team(self.team.id).get(user_id=self.user.id)+ assert state.pending_units == 3+ assert self._select(seed=False) == [(self.team.id, self.user.id)]++ with patch("products.signals.backend.activity_summary.logic.gateway_client", return_value=FakeLLM([ANSWER])):+ result = run_pair(+ ActivitySummaryPairInput(+ team_id=self.team.id,+ user_id=self.user.id,+ min_units=3,+ min_update_interval_hours=24,+ emit_embeddings=False,+ capture_events=False,+ )+ )++ assert result.status == "ok"+ later = NOW + datetime.timedelta(days=2)+ assert self._select(seed=False, now=later) == []+ with (+ time_machine.travel(later, tick=False),+ patch("products.signals.backend.activity_summary.queue.list_user_task_times", return_value=[]),+ ):+ self._changes(self.user, 3, days_ago=-0.1)+ feed_queue(config, later)+ assert self._select(seed=False, now=later) == [(self.team.id, self.user.id)]+def test_an_ok_update_emits_the_embedding_and_an_event_without_the_summary(self) -> None:self._changes(self.user, 3)capture = MagicMock()diff --git a/products/tasks/backend/logic/user_task_summaries.py b/products/tasks/backend/logic/user_task_summaries.pyindex 2de1d2503b8e..84cd2471b33c 100644--- a/products/tasks/backend/logic/user_task_summaries.py+++ b/products/tasks/backend/logic/user_task_summaries.py@@ -118,22 +118,26 @@ def list_user_task_times(start: datetime.datetime,end: datetime.datetime,origin_products: Collection[str],+ exclude_title_prefixes: Collection[str] = (),) -> list[UserTaskTime]:"""The creator and creation time of every non-deleted task in these teams, in [start, end).One query for all users, so a caller can count new tasks per person without a query per person.- Tasks with no creator are left out.+ Tasks with no creator, and tasks whose title starts with one of `exclude_title_prefixes`, are left out."""if not team_ids or not origin_products:return []- rows = Task.objects.filter(+ queryset = Task.objects.filter(team_id__in=list(team_ids),created_by_id__isnull=False,origin_product__in=list(origin_products),created_at__gte=start,created_at__lt=end,deleted=False,- ).values_list("team_id", "created_by_id", "created_at")+ )+ for prefix in exclude_title_prefixes:+ queryset = queryset.exclude(title__startswith=prefix)+ rows = queryset.values_list("team_id", "created_by_id", "created_at")return [UserTaskTime(team_id=team_id, user_id=user_id, created_at=created_at) for team_id, user_id, created_at in rows]