Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): stop serving a stale activity summary after its rebuild skips (#127061)

posthog[bot]deab1507e4d6
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index b81df1805439..4859c1c7e00f 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -256,13 +256,17 @@ def _back_off(
stored: UserActivitySummary | None,
now: datetime.datetime,
max_skip_backoff: datetime.timedelta,
+ *,
+ rebuild: bool,
) -> None:
skips = stored.consecutive_skips if stored is not None else 0
next_eligible_at = now + skip_backoff(skips, max_skip_backoff)
# `update()` leaves `updated_at` alone, so the daily caps do not count a skip as a model call.
if stored is not None:
+ # A skipped rebuild leaves no current summary, so the read stops serving the stale one.
+ status = {"status": UserActivitySummary.Status.SKIPPED} if rebuild else {}
UserActivitySummary.objects.for_team(team_id).filter(id=stored.id).update(
- next_eligible_at=next_eligible_at, consecutive_skips=min(skips + 1, _MAX_SKIP_DOUBLINGS)
+ next_eligible_at=next_eligible_at, consecutive_skips=min(skips + 1, _MAX_SKIP_DOUBLINGS), **status
)
return
UserActivitySummary.objects.for_team(team_id).get_or_create(
@@ -389,6 +393,7 @@ def update_activity_summary(
A failed update sets `next_eligible_at` to `RETRY_BACKOFF` from now. A skip for too few units or for an
empty first profile sets it to `skip_backoff` from now, which doubles with each skip until a good update
resets it. An empty first profile keeps the cost of its model call.
+ A skip of a stale summary's rebuild keeps the stored summary, but sets the status, so the read stops serving it.
A successful update also appends a `UserActivitySummaryVersion` and keeps the latest `max_versions`.
With `mcp_buffer_days`, the update also reads up to that many UTC days of the MCP intent buffer.
"""
@@ -409,7 +414,7 @@ def update_activity_summary(
rendered = _render(team_id, user_id, tasks, activity, views, as_of=now, limits=limits, mcp=window.mcp_records)
if rendered is None or rendered.units < min_units:
if not dry_run:
- _back_off(team_id, user_id, stored, now, max_skip_backoff)
+ _back_off(team_id, user_id, stored, now, max_skip_backoff, rebuild=rebuild)
return ActivitySummaryUpdate(
status=UpdateStatus.SKIPPED_TOO_FEW_UNITS,
rendered_input=rendered.text if rendered else None,
diff --git a/products/signals/backend/activity_summary/read.py b/products/signals/backend/activity_summary/read.py
index 692b14e2f42b..537a416cb13b 100644
--- a/products/signals/backend/activity_summary/read.py
+++ b/products/signals/backend/activity_summary/read.py
@@ -19,11 +19,12 @@ from products.signals.backend.activity_summary.queue import is_team_enrolled_for
from products.signals.backend.models import UserActivitySummary, UserActivitySummaryVersion
DEFAULT_VERSIONS_LIMIT = 5
+_SKIPPED_REBUILD_STATUSES = frozenset({UserActivitySummary.Status.SKIPPED, UserActivitySummary.Status.EMPTY_PROFILE})
class ActivitySummaryStatus(models.TextChoices):
READY = "ready"
- # No row, or no update succeeded yet.
+ # No row, no update succeeded yet, or the rebuild of a stale summary skipped.
NOT_READY = "not_ready"
@@ -113,12 +114,19 @@ def readable_config(team: Team) -> ActivitySummaryConfig | None:
def get_summary(team: Team, user_id: int) -> ActivitySummaryView | None:
- """The user's current summary. A failed later update keeps the last good summary, so it is still ready."""
+ """The user's current summary. A failed later update keeps the last good summary, so it is still ready.
+
+ A stale summary whose rebuild skipped is not ready, because the current inputs gave no summary.
+ """
config = readable_config(team)
if config is None:
return None
row = UserActivitySummary.objects.for_team(team.id).filter(user_id=user_id).first()
- return (row and ready_view(row, config)) or ActivitySummaryView(status=ActivitySummaryStatus.NOT_READY)
+ if row is None or (view := ready_view(row, config)) is None:
+ return ActivitySummaryView(status=ActivitySummaryStatus.NOT_READY)
+ if view.stale and row.status in _SKIPPED_REBUILD_STATUSES:
+ return ActivitySummaryView(status=ActivitySummaryStatus.NOT_READY)
+ return view
def ready_view(row: UserActivitySummary, config: ActivitySummaryConfig) -> ActivitySummaryView | None:
diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.py
index fed378e173ec..33b92b5f672b 100644
--- a/products/signals/backend/test/test_activity_summary_update.py
+++ b/products/signals/backend/test/test_activity_summary_update.py
@@ -14,6 +14,7 @@ from products.dashboards.backend.models.dashboard import Dashboard
from products.dashboards.backend.models.dashboard_tile import DashboardTile
from products.product_analytics.backend.facade.models import Insight
from products.signals.backend.activity_summary import mcp_buffer
+from products.signals.backend.activity_summary.config import ActivitySummaryConfig
from products.signals.backend.activity_summary.inputs import user_activity
from products.signals.backend.activity_summary.logic import (
ActivitySummaryUpdate,
@@ -22,6 +23,7 @@ from products.signals.backend.activity_summary.logic import (
update_activity_summary,
)
from products.signals.backend.activity_summary.prompt import PROMPT_VERSION
+from products.signals.backend.activity_summary.read import ActivitySummaryStatus, get_summary
from products.signals.backend.activity_summary.render import MCP_HEADER, RENDER_VERSION, Product, RenderLimits
from products.signals.backend.models import UserActivitySummary, UserActivitySummaryVersion
from products.signals.backend.test.test_activity_summary_render import FakeLLM
@@ -271,6 +273,51 @@ class TestUpdateActivitySummary(BaseTest):
assert stored is not None
assert (stored.prompt_version, stored.render_version) == (PROMPT_VERSION, RENDER_VERSION)
+ @parameterized.expand(
+ [
+ ("stale, too few units", True, 2, FakeLLM([ANSWER]), ActivitySummaryStatus.NOT_READY),
+ ("stale, empty profile", True, 3, FakeLLM([EMPTY_PROFILE_ANSWER]), ActivitySummaryStatus.NOT_READY),
+ ("stale, failed", True, 3, FakeLLM([(ANSWER, "length")], usage=False), ActivitySummaryStatus.READY),
+ ("current, too few units", False, 2, FakeLLM([ANSWER]), ActivitySummaryStatus.READY),
+ ]
+ )
+ def test_a_skipped_rebuild_of_a_stale_summary_reads_not_ready_until_a_good_build(
+ self, _name: str, stale: bool, changes: int, llm: FakeLLM, read_status: ActivitySummaryStatus
+ ) -> None:
+ old_summary = json.loads(ANSWER) | {"headline": "Old stored summary."}
+ UserActivitySummary.objects.for_team(self.team.id).create(
+ team_id=self.team.id,
+ user_id=self.user.id,
+ status=UserActivitySummary.Status.OK,
+ summary=old_summary,
+ generated_at=NOW - datetime.timedelta(days=2),
+ input_watermark=NOW - datetime.timedelta(days=2),
+ prompt_version=PROMPT_VERSION - 1 if stale else PROMPT_VERSION,
+ render_version=RENDER_VERSION,
+ version=1,
+ )
+ self._changes(changes, days_ago=1)
+ config = ActivitySummaryConfig(enabled=True, enroll_all_approved=True)
+
+ with patch("products.signals.backend.activity_summary.read.read_config", return_value=config):
+ self._update(llm)
+ skipped = get_summary(self.team, self.user.id)
+ self._changes(3, days_ago=0.5)
+ assert self._update(FakeLLM([ANSWER]), force=True).status == UpdateStatus.OK
+ rebuilt = get_summary(self.team, self.user.id)
+
+ stored = self._stored()
+ assert skipped is not None and rebuilt is not None and stored is not None
+ assert skipped.status == read_status
+ if read_status == ActivitySummaryStatus.READY:
+ assert (skipped.headline, skipped.stale) == ("Old stored summary.", stale)
+ assert (rebuilt.status, rebuilt.headline, rebuilt.stale) == (
+ ActivitySummaryStatus.READY,
+ "Builds the signup funnel.",
+ False,
+ )
+ assert stored.version == 2
+
def test_a_failed_update_keeps_the_stored_summary_and_watermark(self) -> None:
self._changes(3, days_ago=2)
self._update(FakeLLM([ANSWER]))