:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(signals): rebuild stale activity summaries from the full window (#121496)
posthog[bot]dd42dd1d7a27
diff --git a/products/signals/backend/activity_summary/config.py b/products/signals/backend/activity_summary/config.pyindex 23283d95ed49..a8980c525423 100644--- a/products/signals/backend/activity_summary/config.py+++ b/products/signals/backend/activity_summary/config.py@@ -4,9 +4,12 @@ The parse fails closed: a missing, malformed or unreadable payload turns the ticfield falls back to its default, and no default is unbounded."""+import datetimeimport dataclassesfrom typing import Any+from django.utils import timezone+import structlogfrom posthog.dataclasses import frozen@@ -37,6 +40,8 @@ class ActivitySummaryConfig:emit_embeddings: bool = Truecapture_events: bool = Truelimits: RenderLimits = DEFAULT_LIMITS+ # Summaries generated before this time are rebuilt from the full window, as after a version change.+ rebuild_before: datetime.datetime | None = NoneDISABLED = ActivitySummaryConfig()@@ -81,6 +86,19 @@ def parse_limits(payload: Any) -> RenderLimits:)+def _timestamp(value: Any) -> datetime.datetime | None:+ if not isinstance(value, str):+ return None+ try:+ parsed = datetime.datetime.fromisoformat(value)+ except ValueError:+ return None+ if parsed.tzinfo is None:+ parsed = parsed.replace(tzinfo=datetime.UTC)+ # A future time would mark every new summary stale again, so it waits until the time passes.+ return parsed if parsed <= timezone.now() else None++def parse_config(payload: Any) -> ActivitySummaryConfig:if not isinstance(payload, dict) or payload.get("enabled") is not True:return DISABLED@@ -100,6 +118,7 @@ def parse_config(payload: Any) -> ActivitySummaryConfig:emit_embeddings=_bool(payload.get("emit_embeddings"), defaults.emit_embeddings),capture_events=_bool(payload.get("capture_events"), defaults.capture_events),limits=parse_limits(payload.get("limits")),+ rebuild_before=_timestamp(payload.get("rebuild_before")),)diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.pyindex a638f8986501..d9373fd40031 100644--- a/products/signals/backend/activity_summary/logic.py+++ b/products/signals/backend/activity_summary/logic.py@@ -3,7 +3,7 @@import datetimeimport dataclassesfrom enum import StrEnum-from typing import Any+from typing import Any, Protocolfrom django.db.models import Valuefrom django.db.models.functions import Coalesce@@ -70,6 +70,30 @@ class ActivitySummaryUpdate:error: str | None = Noneinputs_used: dict[str, Any] | None = Noneinput_watermark: datetime.datetime | None = None+ rebuilt: bool = False+++class StoredSummaryVersion(Protocol):+ @property+ def prompt_version(self) -> int | None: ...++ @property+ def render_version(self) -> int | None: ...++ @property+ def generated_at(self) -> datetime.datetime | None: ...+++def is_stale(stored: StoredSummaryVersion | None, rebuild_before: datetime.datetime | None) -> bool:+ """True when a stored summary came from an older prompt or render, or before `rebuild_before`.++ A stale summary is rebuilt from the full window, so a version change reaches every input.+ """+ if stored is None or stored.generated_at is None:+ return False+ if (stored.prompt_version, stored.render_version) != (PROMPT_VERSION, RENDER_VERSION):+ return True+ return rebuild_before is not None and stored.generated_at < rebuild_beforedef _input_start(watermark: datetime.datetime | None, now: datetime.datetime, window_days: int) -> datetime.datetime:@@ -150,12 +174,14 @@ def update_activity_summary(min_units: int = MIN_UNITS,min_update_interval: datetime.timedelta = MIN_UPDATE_INTERVAL,limits: RenderLimits = DEFAULT_LIMITS,+ rebuild_before: datetime.datetime | None = None,) -> ActivitySummaryUpdate:"""Update the stored activity summary of one user in one project.Reads only the inputs newer than the stored watermark, or the last `limits.window_days` on a first- run. A failed update records the error and keeps the stored summary and the watermark. With- `dry_run`, the model is still called but nothing is written. `force` ignores `next_eligible_at`.+ run or for a stale summary (see `is_stale`). A stale summary is not sent to the model. A failed+ update records the error and keeps the stored summary and the watermark. With `dry_run`, the+ model is still called but nothing is written. `force` ignores `next_eligible_at`.A skip for too few units and a failed update set `next_eligible_at` to `RETRY_BACKOFF` from now."""team = Team.objects.select_related("organization").get(id=team_id)@@ -167,7 +193,8 @@ def update_activity_summary(if not force and stored is not None and stored.next_eligible_at is not None and stored.next_eligible_at > now:return ActivitySummaryUpdate(status=UpdateStatus.SKIPPED_NOT_ELIGIBLE_YET)- start = _input_start(stored.input_watermark if stored else None, now, limits.window_days)+ rebuild = is_stale(stored, rebuild_before)+ start = _input_start(stored.input_watermark if stored and not rebuild else None, now, limits.window_days)tasks = user_tasks(team_id, user_id, start, now)activity = user_activity(team_id, user_id, start, now)views = user_views(team_id, user_id, start, now)@@ -198,7 +225,7 @@ def update_activity_summary("limits": dataclasses.asdict(limits),}watermark = max([task.at for task in tasks] + [item.at for item in activity] + [view.at for view in views])- previous = stored.summary if stored is not None else None+ previous = stored.summary if stored is not None and not rebuild else Nonemodel = DEFAULT_MODELoutcome = summarize(gateway_client(team_id, user_id, now.date()),@@ -221,6 +248,7 @@ def update_activity_summary(error=outcome.error,inputs_used=inputs_used,input_watermark=watermark if outcome.summary is not None else None,+ rebuilt=rebuild,)if dry_run:return resultdiff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.pyindex c499adf3523f..042fce58c7a7 100644--- a/products/signals/backend/activity_summary/queue.py+++ b/products/signals/backend/activity_summary/queue.py@@ -15,7 +15,7 @@ from posthog.models import ActivityLog, Teamfrom products.signals.backend.activity_summary.config import ActivitySummaryConfigfrom products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS-from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX+from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX, is_stalefrom products.signals.backend.activity_summary.prompt import system_promptfrom products.signals.backend.activity_summary.render import (DEFAULT_LIMITS,@@ -47,6 +47,14 @@ class _Stored:input_watermark: datetime.datetime | Nonegenerated_at: datetime.datetime | Nonenext_eligible_at: datetime.datetime | None+ prompt_version: int | None+ render_version: int | None+++@frozen+class _RowCounts:+ new: int = 0+ window: int = 0@frozen@@ -113,20 +121,24 @@ def _enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:def _stored_rows(team_ids: list[int]) -> dict[ActivitySummaryPair, _Stored]:# nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)rows = UserActivitySummary.all_teams.filter(team_id__in=team_ids).values_list(- "team_id", "user_id", "input_watermark", "generated_at", "next_eligible_at"+ "team_id", "user_id", "input_watermark", "generated_at", "next_eligible_at", "prompt_version", "render_version")return {ActivitySummaryPair(team_id=team_id, user_id=user_id): _Stored(- input_watermark=watermark, generated_at=generated_at, next_eligible_at=eligible+ input_watermark=watermark,+ generated_at=generated_at,+ next_eligible_at=eligible,+ prompt_version=prompt_version,+ render_version=render_version,)- for team_id, user_id, watermark, generated_at, eligible in rows+ for team_id, user_id, watermark, generated_at, eligible, prompt_version, render_version in rows}-def _new_activity_rows(+def _activity_rows(team_ids: list[int], window_start: datetime.datetime, now: datetime.datetime-) -> dict[ActivitySummaryPair, int]:- """Activity log rows per pair after the pair's watermark, with the scope filters of `inputs.py`."""+) -> dict[ActivitySummaryPair, _RowCounts]:+ """Activity log rows per pair after the pair's watermark and in the window, with the scope filters of `inputs.py`."""watermark = Subquery(# nosemgrep: idor-lookup-without-team (correlated on the outer row's team_id)UserActivitySummary.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id")).values(@@ -140,57 +152,65 @@ def _new_activity_rows(.exclude(is_system=True).exclude(was_impersonated=True).exclude(client__startswith=SCOUT_CLIENT_PREFIX)- .filter(created_at__gt=Coalesce(watermark, window_start)).values("team_id", "user_id")- .annotate(rows=Count("id"))- .values_list("team_id", "user_id", "rows")+ .annotate(new=Count("id", filter=Q(created_at__gt=Coalesce(watermark, window_start))), window=Count("id"))+ .values_list("team_id", "user_id", "new", "window"))return {- ActivitySummaryPair(team_id=team_id, user_id=user_id): count- for team_id, user_id, count in rows+ ActivitySummaryPair(team_id=team_id, user_id=user_id): _RowCounts(new=new, window=window)+ for team_id, user_id, new, window in rowsif team_id is not None and user_id is not None}-def _new_tasks(+def _task_rows(team_ids: list[int],window_start: datetime.datetime,now: datetime.datetime,stored: dict[ActivitySummaryPair, _Stored],-) -> Counter[ActivitySummaryPair]:- counts: Counter[ActivitySummaryPair] = Counter()+) -> dict[ActivitySummaryPair, _RowCounts]:+ """Tasks per pair after the pair's watermark and in the window."""+ new: Counter[ActivitySummaryPair] = Counter()+ window: Counter[ActivitySummaryPair] = Counter()for task in list_user_task_times(team_ids=team_ids, start=window_start, end=now, origin_products=TASK_ORIGIN_PRODUCTS):pair = ActivitySummaryPair(team_id=task.team_id, user_id=task.user_id)+ window[pair] += 1row = stored.get(pair)if row is None or row.input_watermark is None or task.created_at > row.input_watermark:- counts[pair] += 1- return counts+ new[pair] += 1+ return {pair: _RowCounts(new=new[pair], window=count) for pair, count in window.items()}def select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: int) -> list[ActivitySummaryPair]:"""The due pairs of the enrolled teams, highest priority first, cut to `limit`.A pair is due when it has a task or an activity log row after its watermark, or in the last- `window_days` with no watermark, and its `next_eligible_at` is empty or past. Pairs with no- summary come first, then the pairs with the most new rows, then the oldest summaries.+ `window_days` with no watermark or a stale summary, and its `next_eligible_at` is empty or past.+ Pairs with no summary or a stale summary come first, then the pairs with the most new rows,+ then the oldest summaries."""team_ids = _enrolled_team_ids(config)if not team_ids or limit <= 0:return []window_start = now - datetime.timedelta(days=config.limits.window_days)stored = _stored_rows(team_ids)- new_rows = Counter(_new_activity_rows(team_ids, window_start, now)) + _new_tasks(- team_ids, window_start, now, stored- )+ activity_rows = _activity_rows(team_ids, window_start, now)+ task_rows = _task_rows(team_ids, window_start, now, stored)due: list[tuple[bool, int, datetime.datetime, ActivitySummaryPair]] = []- for pair, count in new_rows.items():+ for pair in activity_rows.keys() | task_rows.keys():row = stored.get(pair)if row is not None and row.next_eligible_at is not None and row.next_eligible_at > now:continue+ activity = activity_rows.get(pair, _RowCounts())+ tasks = task_rows.get(pair, _RowCounts())+ stale = is_stale(row, config.rebuild_before)+ count = activity.window + tasks.window if stale else activity.new + tasks.new+ if not count:+ continuegenerated_at = row.generated_at if row is not None else None- due.append((generated_at is not None, -count, generated_at or window_start, pair))+ due.append((generated_at is not None and not stale, -count, generated_at or window_start, pair))due.sort()return [pair for *_, pair in due[:limit]]diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.pyindex b4cb7bd22a4c..24cba59571cf 100644--- a/products/signals/backend/activity_summary/tick.py+++ b/products/signals/backend/activity_summary/tick.py@@ -88,6 +88,7 @@ class ActivitySummaryTickPlan:stop_reason: str | None# A default, so a plan that a workflow history recorded before this field existed still decodes.limits: RenderLimits = DEFAULT_LIMITS+ rebuild_before: datetime.datetime | None = None@frozen@@ -100,6 +101,7 @@ class ActivitySummaryPairInput:capture_events: bool# A default, so a pair input that a tick queued before this field existed still decodes.limits: RenderLimits = DEFAULT_LIMITS+ rebuild_before: datetime.datetime | None = None@frozen@@ -157,6 +159,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickPcapture_events=config.capture_events,limits=config.limits,stop_reason=stop_reason,+ rebuild_before=config.rebuild_before,)@@ -206,6 +209,7 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate"cost_usd": update.cost_usd,"latency_ms": update.latency_ms,"inputs_used": update.inputs_used,+ "rebuilt": update.rebuilt,},groups={"organization": str(team.organization_id), "project": str(team.uuid)},)@@ -219,6 +223,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> Amin_units=input.min_units,min_update_interval=datetime.timedelta(hours=input.min_update_interval_hours),limits=input.limits,+ rebuild_before=input.rebuild_before,)called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)except Exception as error:@@ -333,6 +338,7 @@ class ActivitySummaryTickWorkflow:emit_embeddings=plan.emit_embeddings,capture_events=plan.capture_events,limits=plan.limits,+ rebuild_before=plan.rebuild_before,),start_to_close_timeout=_PAIR_TIMEOUT,heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.pyindex 2c63ce12c416..ed693a183ae7 100644--- a/products/signals/backend/test/test_activity_summary_tick.py+++ b/products/signals/backend/test/test_activity_summary_tick.py@@ -26,6 +26,7 @@ from products.signals.backend.activity_summary import (tick,)from products.signals.backend.activity_summary.config import ActivitySummaryConfig, parse_config+from products.signals.backend.activity_summary.prompt import PROMPT_VERSIONfrom products.signals.backend.activity_summary.queue import (ActivitySummaryPair,DailyUsage,@@ -33,7 +34,7 @@ from products.signals.backend.activity_summary.queue import (max_update_cost_usd,select_pairs,)-from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, MAX_LIMITS, RenderLimits+from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, MAX_LIMITS, RENDER_VERSION, RenderLimitsfrom products.signals.backend.activity_summary.schedule import (ACTIVITY_SUMMARY_TICK_SCHEDULE_ID,create_activity_summary_tick_schedule,@@ -87,6 +88,7 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = ["min_units": None,"min_update_interval_hours": 0,"emit_embeddings": "no",+ "rebuild_before": "not a time",},ActivitySummaryConfig(enabled=True, organization_ids=("org-a",), skip_team_ids=(3,)),),@@ -98,6 +100,7 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = ["max_cost_usd_per_day": 1,"min_update_interval_hours": 12,"capture_events": False,+ "rebuild_before": "2026-10-01T00:00:00",},ActivitySummaryConfig(enabled=True,@@ -105,6 +108,7 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [max_cost_usd_per_day=1.0,min_update_interval_hours=12.0,capture_events=False,+ rebuild_before=datetime.datetime(2026, 10, 1, tzinfo=datetime.UTC),),),(@@ -125,10 +129,16 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [),),("limits that are not an object", {"enabled": True, "limits": [5]}, ActivitySummaryConfig(enabled=True)),+ (+ "a future rebuild_before waits",+ {"enabled": True, "rebuild_before": "2026-10-20T00:00:00+00:00"},+ ActivitySummaryConfig(enabled=True),+ ),]@parameterized.expand(_CONFIG_CASES)+@time_machine.travel(NOW, tick=False)def test_parse_config(_name: str, payload: Any, expected: ActivitySummaryConfig) -> None:assert parse_config(payload) == expected@@ -157,7 +167,10 @@ class TestActivitySummaryQueue(BaseTest):def _summary(self, user: User, **fields: Any) -> None:UserActivitySummary.objects.for_team(self.team.id).create(- team_id=self.team.id, user_id=user.id, status=UserActivitySummary.Status.OK, **fields+ team_id=self.team.id,+ user_id=user.id,+ status=UserActivitySummary.Status.OK,+ **({"prompt_version": PROMPT_VERSION, "render_version": RENDER_VERSION} | fields),)def _select(self, config: ActivitySummaryConfig | None = None, tasks: list[UserTaskTime] | None = None):@@ -190,6 +203,29 @@ class TestActivitySummaryQueue(BaseTest):(self.team.id, fresh.id),]+ def test_a_stale_summary_is_due_without_new_rows_and_ranks_with_cold_starts(self) -> None:+ current, old_version, before_rebuild, busy = (+ self._user(f"{name}@example.com") for name in ("current", "old", "rebuild", "busy")+ )+ summary_fields = {+ "generated_at": NOW - datetime.timedelta(days=2),+ "input_watermark": NOW - datetime.timedelta(days=2),+ }+ self._summary(current, **summary_fields)+ self._summary(old_version, **(summary_fields | {"prompt_version": PROMPT_VERSION - 1}))+ self._summary(before_rebuild, **(summary_fields | {"generated_at": NOW - datetime.timedelta(days=3)}))+ self._summary(busy, **summary_fields)+ for user in (current, old_version, before_rebuild):+ self._changes(user, 1, days_ago=5)+ self._changes(busy, 3)++ config = self._config(rebuild_before=NOW - datetime.timedelta(days=2, hours=12))+ assert self._select(config) == [+ (self.team.id, before_rebuild.id),+ (self.team.id, old_version.id),+ (self.team.id, busy.id),+ ]+def test_leaves_out_pairs_that_are_not_due_or_not_enrolled(self) -> None:waiting, scout_only, impersonated, skipped_team = (self._user(f"{name}@example.com") for name in ("waiting", "scout", "impersonated", "skipped")diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.pyindex c7d7a198a08c..0cefbd6304fc 100644--- a/products/signals/backend/test/test_activity_summary_update.py+++ b/products/signals/backend/test/test_activity_summary_update.py@@ -15,7 +15,8 @@ from products.dashboards.backend.models.dashboard_tile import DashboardTilefrom products.product_analytics.backend.facade.models import Insightfrom products.signals.backend.activity_summary.inputs import user_activityfrom products.signals.backend.activity_summary.logic import ActivitySummaryUpdate, UpdateStatus, update_activity_summary-from products.signals.backend.activity_summary.render import RenderLimits+from products.signals.backend.activity_summary.prompt import PROMPT_VERSION+from products.signals.backend.activity_summary.render import RENDER_VERSION, RenderLimitsfrom products.signals.backend.models import UserActivitySummaryfrom products.signals.backend.test.test_activity_summary_render import FakeLLM@@ -158,6 +159,41 @@ class TestUpdateActivitySummary(BaseTest):]assert result.inputs_used["drops"] == {"grouped_into_parent": 2}+ @parameterized.expand(+ [+ ("an older prompt version", {"prompt_version": PROMPT_VERSION - 1}, {}, True),+ ("an older render version", {"render_version": RENDER_VERSION - 1}, {}, True),+ ("a rebuild_before after generated_at", {}, {"rebuild_before": NOW - datetime.timedelta(hours=1)}, True),+ ("a rebuild_before before generated_at", {}, {"rebuild_before": NOW - datetime.timedelta(days=3)}, False),+ ]+ )+ def test_a_stale_summary_rebuilds_from_the_full_window(+ self, _name: str, stored_fields: dict[str, Any], kwargs: dict[str, Any], rebuilt: bool+ ) -> None:+ for index in range(3):+ self._change(f"Old dashboard {index}", days_ago=5 + index / 100)+ UserActivitySummary.objects.for_team(self.team.id).create(+ team_id=self.team.id,+ user_id=self.user.id,+ status=UserActivitySummary.Status.OK,+ summary=json.loads(ANSWER) | {"headline": "Old stored summary."},+ generated_at=NOW - datetime.timedelta(days=2),+ input_watermark=NOW - datetime.timedelta(days=2),+ **({"prompt_version": PROMPT_VERSION, "render_version": RENDER_VERSION} | stored_fields),+ )+ self._changes(3, days_ago=1)+ llm = FakeLLM([ANSWER])++ result = self._update(llm, **kwargs)++ assert (result.status, result.rebuilt) == (UpdateStatus.OK, rebuilt)+ sent = json.dumps(llm.requests[0]["messages"])+ assert ("Old stored summary." not in sent) == rebuilt+ assert ("Old dashboard" in sent) == rebuilt+ stored = self._stored()+ assert stored is not None+ assert (stored.prompt_version, stored.render_version) == (PROMPT_VERSION, RENDER_VERSION)+def test_a_failed_update_keeps_the_stored_summary_and_watermark(self) -> None:self._changes(3, days_ago=2)self._update(FakeLLM([ANSWER]))