Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(signals): rebuild stale activity summaries from the full window (#121496)

posthog[bot]dd42dd1d7a27
diff --git a/products/signals/backend/activity_summary/config.py b/products/signals/backend/activity_summary/config.py
index 23283d95ed49..a8980c525423 100644
--- a/products/signals/backend/activity_summary/config.py
+++ b/products/signals/backend/activity_summary/config.py
@@ -4,9 +4,12 @@ The parse fails closed: a missing, malformed or unreadable payload turns the tic
field falls back to its default, and no default is unbounded.
"""
+import datetime
import dataclasses
from typing import Any
+from django.utils import timezone
+
import structlog
from posthog.dataclasses import frozen
@@ -37,6 +40,8 @@ class ActivitySummaryConfig:
emit_embeddings: bool = True
capture_events: bool = True
limits: RenderLimits = DEFAULT_LIMITS
+ # Summaries generated before this time are rebuilt from the full window, as after a version change.
+ rebuild_before: datetime.datetime | None = None
DISABLED = ActivitySummaryConfig()
@@ -81,6 +86,19 @@ def parse_limits(payload: Any) -> RenderLimits:
)
+def _timestamp(value: Any) -> datetime.datetime | None:
+ if not isinstance(value, str):
+ return None
+ try:
+ parsed = datetime.datetime.fromisoformat(value)
+ except ValueError:
+ return None
+ if parsed.tzinfo is None:
+ parsed = parsed.replace(tzinfo=datetime.UTC)
+ # A future time would mark every new summary stale again, so it waits until the time passes.
+ return parsed if parsed <= timezone.now() else None
+
+
def parse_config(payload: Any) -> ActivitySummaryConfig:
if not isinstance(payload, dict) or payload.get("enabled") is not True:
return DISABLED
@@ -100,6 +118,7 @@ def parse_config(payload: Any) -> ActivitySummaryConfig:
emit_embeddings=_bool(payload.get("emit_embeddings"), defaults.emit_embeddings),
capture_events=_bool(payload.get("capture_events"), defaults.capture_events),
limits=parse_limits(payload.get("limits")),
+ rebuild_before=_timestamp(payload.get("rebuild_before")),
)
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index a638f8986501..d9373fd40031 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -3,7 +3,7 @@
import datetime
import dataclasses
from enum import StrEnum
-from typing import Any
+from typing import Any, Protocol
from django.db.models import Value
from django.db.models.functions import Coalesce
@@ -70,6 +70,30 @@ class ActivitySummaryUpdate:
error: str | None = None
inputs_used: dict[str, Any] | None = None
input_watermark: datetime.datetime | None = None
+ rebuilt: bool = False
+
+
+class StoredSummaryVersion(Protocol):
+ @property
+ def prompt_version(self) -> int | None: ...
+
+ @property
+ def render_version(self) -> int | None: ...
+
+ @property
+ def generated_at(self) -> datetime.datetime | None: ...
+
+
+def is_stale(stored: StoredSummaryVersion | None, rebuild_before: datetime.datetime | None) -> bool:
+ """True when a stored summary came from an older prompt or render, or before `rebuild_before`.
+
+ A stale summary is rebuilt from the full window, so a version change reaches every input.
+ """
+ if stored is None or stored.generated_at is None:
+ return False
+ if (stored.prompt_version, stored.render_version) != (PROMPT_VERSION, RENDER_VERSION):
+ return True
+ return rebuild_before is not None and stored.generated_at < rebuild_before
def _input_start(watermark: datetime.datetime | None, now: datetime.datetime, window_days: int) -> datetime.datetime:
@@ -150,12 +174,14 @@ def update_activity_summary(
min_units: int = MIN_UNITS,
min_update_interval: datetime.timedelta = MIN_UPDATE_INTERVAL,
limits: RenderLimits = DEFAULT_LIMITS,
+ rebuild_before: datetime.datetime | None = None,
) -> ActivitySummaryUpdate:
"""Update the stored activity summary of one user in one project.
Reads only the inputs newer than the stored watermark, or the last `limits.window_days` on a first
- run. A failed update records the error and keeps the stored summary and the watermark. With
- `dry_run`, the model is still called but nothing is written. `force` ignores `next_eligible_at`.
+ run or for a stale summary (see `is_stale`). A stale summary is not sent to the model. A failed
+ update records the error and keeps the stored summary and the watermark. With `dry_run`, the
+ model is still called but nothing is written. `force` ignores `next_eligible_at`.
A skip for too few units and a failed update set `next_eligible_at` to `RETRY_BACKOFF` from now.
"""
team = Team.objects.select_related("organization").get(id=team_id)
@@ -167,7 +193,8 @@ def update_activity_summary(
if not force and stored is not None and stored.next_eligible_at is not None and stored.next_eligible_at > now:
return ActivitySummaryUpdate(status=UpdateStatus.SKIPPED_NOT_ELIGIBLE_YET)
- start = _input_start(stored.input_watermark if stored else None, now, limits.window_days)
+ rebuild = is_stale(stored, rebuild_before)
+ start = _input_start(stored.input_watermark if stored and not rebuild else None, now, limits.window_days)
tasks = user_tasks(team_id, user_id, start, now)
activity = user_activity(team_id, user_id, start, now)
views = user_views(team_id, user_id, start, now)
@@ -198,7 +225,7 @@ def update_activity_summary(
"limits": dataclasses.asdict(limits),
}
watermark = max([task.at for task in tasks] + [item.at for item in activity] + [view.at for view in views])
- previous = stored.summary if stored is not None else None
+ previous = stored.summary if stored is not None and not rebuild else None
model = DEFAULT_MODEL
outcome = summarize(
gateway_client(team_id, user_id, now.date()),
@@ -221,6 +248,7 @@ def update_activity_summary(
error=outcome.error,
inputs_used=inputs_used,
input_watermark=watermark if outcome.summary is not None else None,
+ rebuilt=rebuild,
)
if dry_run:
return result
diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.py
index c499adf3523f..042fce58c7a7 100644
--- a/products/signals/backend/activity_summary/queue.py
+++ b/products/signals/backend/activity_summary/queue.py
@@ -15,7 +15,7 @@ from posthog.models import ActivityLog, Team
from products.signals.backend.activity_summary.config import ActivitySummaryConfig
from products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS
-from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX
+from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX, is_stale
from products.signals.backend.activity_summary.prompt import system_prompt
from products.signals.backend.activity_summary.render import (
DEFAULT_LIMITS,
@@ -47,6 +47,14 @@ class _Stored:
input_watermark: datetime.datetime | None
generated_at: datetime.datetime | None
next_eligible_at: datetime.datetime | None
+ prompt_version: int | None
+ render_version: int | None
+
+
+@frozen
+class _RowCounts:
+ new: int = 0
+ window: int = 0
@frozen
@@ -113,20 +121,24 @@ def _enrolled_team_ids(config: ActivitySummaryConfig) -> list[int]:
def _stored_rows(team_ids: list[int]) -> dict[ActivitySummaryPair, _Stored]:
# nosemgrep: idor-lookup-without-team (system tick with no user input; filtered to the enrolled teams)
rows = UserActivitySummary.all_teams.filter(team_id__in=team_ids).values_list(
- "team_id", "user_id", "input_watermark", "generated_at", "next_eligible_at"
+ "team_id", "user_id", "input_watermark", "generated_at", "next_eligible_at", "prompt_version", "render_version"
)
return {
ActivitySummaryPair(team_id=team_id, user_id=user_id): _Stored(
- input_watermark=watermark, generated_at=generated_at, next_eligible_at=eligible
+ input_watermark=watermark,
+ generated_at=generated_at,
+ next_eligible_at=eligible,
+ prompt_version=prompt_version,
+ render_version=render_version,
)
- for team_id, user_id, watermark, generated_at, eligible in rows
+ for team_id, user_id, watermark, generated_at, eligible, prompt_version, render_version in rows
}
-def _new_activity_rows(
+def _activity_rows(
team_ids: list[int], window_start: datetime.datetime, now: datetime.datetime
-) -> dict[ActivitySummaryPair, int]:
- """Activity log rows per pair after the pair's watermark, with the scope filters of `inputs.py`."""
+) -> dict[ActivitySummaryPair, _RowCounts]:
+ """Activity log rows per pair after the pair's watermark and in the window, with the scope filters of `inputs.py`."""
watermark = Subquery(
# nosemgrep: idor-lookup-without-team (correlated on the outer row's team_id)
UserActivitySummary.all_teams.filter(team_id=OuterRef("team_id"), user_id=OuterRef("user_id")).values(
@@ -140,57 +152,65 @@ def _new_activity_rows(
.exclude(is_system=True)
.exclude(was_impersonated=True)
.exclude(client__startswith=SCOUT_CLIENT_PREFIX)
- .filter(created_at__gt=Coalesce(watermark, window_start))
.values("team_id", "user_id")
- .annotate(rows=Count("id"))
- .values_list("team_id", "user_id", "rows")
+ .annotate(new=Count("id", filter=Q(created_at__gt=Coalesce(watermark, window_start))), window=Count("id"))
+ .values_list("team_id", "user_id", "new", "window")
)
return {
- ActivitySummaryPair(team_id=team_id, user_id=user_id): count
- for team_id, user_id, count in rows
+ ActivitySummaryPair(team_id=team_id, user_id=user_id): _RowCounts(new=new, window=window)
+ for team_id, user_id, new, window in rows
if team_id is not None and user_id is not None
}
-def _new_tasks(
+def _task_rows(
team_ids: list[int],
window_start: datetime.datetime,
now: datetime.datetime,
stored: dict[ActivitySummaryPair, _Stored],
-) -> Counter[ActivitySummaryPair]:
- counts: Counter[ActivitySummaryPair] = Counter()
+) -> dict[ActivitySummaryPair, _RowCounts]:
+ """Tasks per pair after the pair's watermark and in the window."""
+ new: Counter[ActivitySummaryPair] = Counter()
+ window: Counter[ActivitySummaryPair] = Counter()
for task in list_user_task_times(
team_ids=team_ids, start=window_start, end=now, origin_products=TASK_ORIGIN_PRODUCTS
):
pair = ActivitySummaryPair(team_id=task.team_id, user_id=task.user_id)
+ window[pair] += 1
row = stored.get(pair)
if row is None or row.input_watermark is None or task.created_at > row.input_watermark:
- counts[pair] += 1
- return counts
+ new[pair] += 1
+ return {pair: _RowCounts(new=new[pair], window=count) for pair, count in window.items()}
def select_pairs(config: ActivitySummaryConfig, now: datetime.datetime, limit: int) -> list[ActivitySummaryPair]:
"""The due pairs of the enrolled teams, highest priority first, cut to `limit`.
A pair is due when it has a task or an activity log row after its watermark, or in the last
- `window_days` with no watermark, and its `next_eligible_at` is empty or past. Pairs with no
- summary come first, then the pairs with the most new rows, then the oldest summaries.
+ `window_days` with no watermark or a stale summary, and its `next_eligible_at` is empty or past.
+ Pairs with no summary or a stale summary come first, then the pairs with the most new rows,
+ then the oldest summaries.
"""
team_ids = _enrolled_team_ids(config)
if not team_ids or limit <= 0:
return []
window_start = now - datetime.timedelta(days=config.limits.window_days)
stored = _stored_rows(team_ids)
- new_rows = Counter(_new_activity_rows(team_ids, window_start, now)) + _new_tasks(
- team_ids, window_start, now, stored
- )
+ activity_rows = _activity_rows(team_ids, window_start, now)
+ task_rows = _task_rows(team_ids, window_start, now, stored)
due: list[tuple[bool, int, datetime.datetime, ActivitySummaryPair]] = []
- for pair, count in new_rows.items():
+ for pair in activity_rows.keys() | task_rows.keys():
row = stored.get(pair)
if row is not None and row.next_eligible_at is not None and row.next_eligible_at > now:
continue
+ activity = activity_rows.get(pair, _RowCounts())
+ tasks = task_rows.get(pair, _RowCounts())
+ stale = is_stale(row, config.rebuild_before)
+ count = activity.window + tasks.window if stale else activity.new + tasks.new
+ if not count:
+ continue
generated_at = row.generated_at if row is not None else None
- due.append((generated_at is not None, -count, generated_at or window_start, pair))
+ due.append((generated_at is not None and not stale, -count, generated_at or window_start, pair))
due.sort()
return [pair for *_, pair in due[:limit]]
diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.py
index b4cb7bd22a4c..24cba59571cf 100644
--- a/products/signals/backend/activity_summary/tick.py
+++ b/products/signals/backend/activity_summary/tick.py
@@ -88,6 +88,7 @@ class ActivitySummaryTickPlan:
stop_reason: str | None
# A default, so a plan that a workflow history recorded before this field existed still decodes.
limits: RenderLimits = DEFAULT_LIMITS
+ rebuild_before: datetime.datetime | None = None
@frozen
@@ -100,6 +101,7 @@ class ActivitySummaryPairInput:
capture_events: bool
# A default, so a pair input that a tick queued before this field existed still decodes.
limits: RenderLimits = DEFAULT_LIMITS
+ rebuild_before: datetime.datetime | None = None
@frozen
@@ -157,6 +159,7 @@ def plan_tick(limit: int | None, now: datetime.datetime) -> ActivitySummaryTickP
capture_events=config.capture_events,
limits=config.limits,
stop_reason=stop_reason,
+ rebuild_before=config.rebuild_before,
)
@@ -206,6 +209,7 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate
"cost_usd": update.cost_usd,
"latency_ms": update.latency_ms,
"inputs_used": update.inputs_used,
+ "rebuilt": update.rebuilt,
},
groups={"organization": str(team.organization_id), "project": str(team.uuid)},
)
@@ -219,6 +223,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A
min_units=input.min_units,
min_update_interval=datetime.timedelta(hours=input.min_update_interval_hours),
limits=input.limits,
+ rebuild_before=input.rebuild_before,
)
called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)
except Exception as error:
@@ -333,6 +338,7 @@ class ActivitySummaryTickWorkflow:
emit_embeddings=plan.emit_embeddings,
capture_events=plan.capture_events,
limits=plan.limits,
+ rebuild_before=plan.rebuild_before,
),
start_to_close_timeout=_PAIR_TIMEOUT,
heartbeat_timeout=_PAIR_HEARTBEAT_TIMEOUT,
diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.py
index 2c63ce12c416..ed693a183ae7 100644
--- a/products/signals/backend/test/test_activity_summary_tick.py
+++ b/products/signals/backend/test/test_activity_summary_tick.py
@@ -26,6 +26,7 @@ from products.signals.backend.activity_summary import (
tick,
)
from products.signals.backend.activity_summary.config import ActivitySummaryConfig, parse_config
+from products.signals.backend.activity_summary.prompt import PROMPT_VERSION
from products.signals.backend.activity_summary.queue import (
ActivitySummaryPair,
DailyUsage,
@@ -33,7 +34,7 @@ from products.signals.backend.activity_summary.queue import (
max_update_cost_usd,
select_pairs,
)
-from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, MAX_LIMITS, RenderLimits
+from products.signals.backend.activity_summary.render import DEFAULT_LIMITS, MAX_LIMITS, RENDER_VERSION, RenderLimits
from products.signals.backend.activity_summary.schedule import (
ACTIVITY_SUMMARY_TICK_SCHEDULE_ID,
create_activity_summary_tick_schedule,
@@ -87,6 +88,7 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
"min_units": None,
"min_update_interval_hours": 0,
"emit_embeddings": "no",
+ "rebuild_before": "not a time",
},
ActivitySummaryConfig(enabled=True, organization_ids=("org-a",), skip_team_ids=(3,)),
),
@@ -98,6 +100,7 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
"max_cost_usd_per_day": 1,
"min_update_interval_hours": 12,
"capture_events": False,
+ "rebuild_before": "2026-10-01T00:00:00",
},
ActivitySummaryConfig(
enabled=True,
@@ -105,6 +108,7 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
max_cost_usd_per_day=1.0,
min_update_interval_hours=12.0,
capture_events=False,
+ rebuild_before=datetime.datetime(2026, 10, 1, tzinfo=datetime.UTC),
),
),
(
@@ -125,10 +129,16 @@ _CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
),
),
("limits that are not an object", {"enabled": True, "limits": [5]}, ActivitySummaryConfig(enabled=True)),
+ (
+ "a future rebuild_before waits",
+ {"enabled": True, "rebuild_before": "2026-10-20T00:00:00+00:00"},
+ ActivitySummaryConfig(enabled=True),
+ ),
]
@parameterized.expand(_CONFIG_CASES)
+@time_machine.travel(NOW, tick=False)
def test_parse_config(_name: str, payload: Any, expected: ActivitySummaryConfig) -> None:
assert parse_config(payload) == expected
@@ -157,7 +167,10 @@ class TestActivitySummaryQueue(BaseTest):
def _summary(self, user: User, **fields: Any) -> None:
UserActivitySummary.objects.for_team(self.team.id).create(
- team_id=self.team.id, user_id=user.id, status=UserActivitySummary.Status.OK, **fields
+ team_id=self.team.id,
+ user_id=user.id,
+ status=UserActivitySummary.Status.OK,
+ **({"prompt_version": PROMPT_VERSION, "render_version": RENDER_VERSION} | fields),
)
def _select(self, config: ActivitySummaryConfig | None = None, tasks: list[UserTaskTime] | None = None):
@@ -190,6 +203,29 @@ class TestActivitySummaryQueue(BaseTest):
(self.team.id, fresh.id),
]
+ def test_a_stale_summary_is_due_without_new_rows_and_ranks_with_cold_starts(self) -> None:
+ current, old_version, before_rebuild, busy = (
+ self._user(f"{name}@example.com") for name in ("current", "old", "rebuild", "busy")
+ )
+ summary_fields = {
+ "generated_at": NOW - datetime.timedelta(days=2),
+ "input_watermark": NOW - datetime.timedelta(days=2),
+ }
+ self._summary(current, **summary_fields)
+ self._summary(old_version, **(summary_fields | {"prompt_version": PROMPT_VERSION - 1}))
+ self._summary(before_rebuild, **(summary_fields | {"generated_at": NOW - datetime.timedelta(days=3)}))
+ self._summary(busy, **summary_fields)
+ for user in (current, old_version, before_rebuild):
+ self._changes(user, 1, days_ago=5)
+ self._changes(busy, 3)
+
+ config = self._config(rebuild_before=NOW - datetime.timedelta(days=2, hours=12))
+ assert self._select(config) == [
+ (self.team.id, before_rebuild.id),
+ (self.team.id, old_version.id),
+ (self.team.id, busy.id),
+ ]
+
def test_leaves_out_pairs_that_are_not_due_or_not_enrolled(self) -> None:
waiting, scout_only, impersonated, skipped_team = (
self._user(f"{name}@example.com") for name in ("waiting", "scout", "impersonated", "skipped")
diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.py
index c7d7a198a08c..0cefbd6304fc 100644
--- a/products/signals/backend/test/test_activity_summary_update.py
+++ b/products/signals/backend/test/test_activity_summary_update.py
@@ -15,7 +15,8 @@ from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.product_analytics.backend.facade.models import Insight
from products.signals.backend.activity_summary.inputs import user_activity
from products.signals.backend.activity_summary.logic import ActivitySummaryUpdate, UpdateStatus, update_activity_summary
-from products.signals.backend.activity_summary.render import RenderLimits
+from products.signals.backend.activity_summary.prompt import PROMPT_VERSION
+from products.signals.backend.activity_summary.render import RENDER_VERSION, RenderLimits
from products.signals.backend.models import UserActivitySummary
from products.signals.backend.test.test_activity_summary_render import FakeLLM
@@ -158,6 +159,41 @@ class TestUpdateActivitySummary(BaseTest):
]
assert result.inputs_used["drops"] == {"grouped_into_parent": 2}
+ @parameterized.expand(
+ [
+ ("an older prompt version", {"prompt_version": PROMPT_VERSION - 1}, {}, True),
+ ("an older render version", {"render_version": RENDER_VERSION - 1}, {}, True),
+ ("a rebuild_before after generated_at", {}, {"rebuild_before": NOW - datetime.timedelta(hours=1)}, True),
+ ("a rebuild_before before generated_at", {}, {"rebuild_before": NOW - datetime.timedelta(days=3)}, False),
+ ]
+ )
+ def test_a_stale_summary_rebuilds_from_the_full_window(
+ self, _name: str, stored_fields: dict[str, Any], kwargs: dict[str, Any], rebuilt: bool
+ ) -> None:
+ for index in range(3):
+ self._change(f"Old dashboard {index}", days_ago=5 + index / 100)
+ UserActivitySummary.objects.for_team(self.team.id).create(
+ team_id=self.team.id,
+ user_id=self.user.id,
+ status=UserActivitySummary.Status.OK,
+ summary=json.loads(ANSWER) | {"headline": "Old stored summary."},
+ generated_at=NOW - datetime.timedelta(days=2),
+ input_watermark=NOW - datetime.timedelta(days=2),
+ **({"prompt_version": PROMPT_VERSION, "render_version": RENDER_VERSION} | stored_fields),
+ )
+ self._changes(3, days_ago=1)
+ llm = FakeLLM([ANSWER])
+
+ result = self._update(llm, **kwargs)
+
+ assert (result.status, result.rebuilt) == (UpdateStatus.OK, rebuilt)
+ sent = json.dumps(llm.requests[0]["messages"])
+ assert ("Old stored summary." not in sent) == rebuilt
+ assert ("Old dashboard" in sent) == rebuilt
+ stored = self._stored()
+ assert stored is not None
+ assert (stored.prompt_version, stored.render_version) == (PROMPT_VERSION, RENDER_VERSION)
+
def test_a_failed_update_keeps_the_stored_summary_and_watermark(self) -> None:
self._changes(3, days_ago=2)
self._update(FakeLLM([ANSWER]))