Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): back off and record activity summary pairs that raise (#121069)

posthog[bot]ee344c7dc6c2
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index 61c96115e0f6..108488d6315a 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -4,6 +4,8 @@ import datetime
from enum import StrEnum
from typing import Any
+from django.db.models import Value
+from django.db.models.functions import Coalesce
from django.utils import timezone
from posthog.dataclasses import frozen
@@ -26,10 +28,21 @@ MIN_UNITS = 3
MIN_UPDATE_INTERVAL = datetime.timedelta(days=1)
# A pair that skipped or failed waits this long, so it does not take the queue head on every tick.
RETRY_BACKOFF = datetime.timedelta(hours=6)
+# The error prefixes of an update that raised before it stored a result, before and after the model call.
+EXCEPTION_ERROR_PREFIX = "exception: "
+EXCEPTION_AFTER_MODEL_CALL_ERROR_PREFIX = "exception_after_model_call: "
# The watermark is inclusive in the stored summary, so the next read starts just after it.
_WATERMARK_STEP = datetime.timedelta(microseconds=1)
+class UpdateAfterModelCallError(Exception):
+ """An update raised after the model call, so the call counts against the daily caps."""
+
+ def __init__(self, cost_usd: float | None) -> None:
+ super().__init__("activity summary update raised after the model call")
+ self.cost_usd = cost_usd
+
+
class UpdateStatus(StrEnum):
OK = "ok"
FAILED = "failed"
@@ -71,6 +84,56 @@ def _back_off(
)
+def record_update_exception(team_id: int, user_id: int, error: BaseException) -> ActivitySummaryUpdate:
+ """Store an update that raised as failed, keep the stored summary and watermark, and back the pair off.
+
+ An update that raised after the model call keeps its cost. A missing cost counts at the bound.
+ An update that raised before the model call keeps the cost of the earlier update.
+ """
+ next_eligible_at = timezone.now() + RETRY_BACKOFF
+ if isinstance(error, UpdateAfterModelCallError):
+ message = f"{EXCEPTION_AFTER_MODEL_CALL_ERROR_PREFIX}{type(error.__cause__).__name__}"
+ UserActivitySummary.objects.for_team(team_id).update_or_create(
+ team_id=team_id,
+ user_id=user_id,
+ defaults={
+ "status": UserActivitySummary.Status.FAILED,
+ "error": message,
+ "tokens_in": None,
+ "tokens_out": None,
+ "cost_usd": error.cost_usd,
+ "latency_ms": None,
+ "next_eligible_at": next_eligible_at,
+ },
+ )
+ return ActivitySummaryUpdate(status=UpdateStatus.FAILED, error=message, cost_usd=error.cost_usd)
+
+ message = f"{EXCEPTION_ERROR_PREFIX}{type(error).__name__}"
+ # `update()` leaves `updated_at` and a paid cost alone, so the daily caps still count an earlier update.
+ updated = (
+ UserActivitySummary.objects.for_team(team_id)
+ .filter(user_id=user_id)
+ .update(
+ status=UserActivitySummary.Status.FAILED,
+ error=message,
+ cost_usd=Coalesce("cost_usd", Value(0.0)),
+ next_eligible_at=next_eligible_at,
+ )
+ )
+ if not updated:
+ UserActivitySummary.objects.for_team(team_id).get_or_create(
+ team_id=team_id,
+ user_id=user_id,
+ defaults={
+ "status": UserActivitySummary.Status.FAILED,
+ "error": message,
+ "cost_usd": 0.0,
+ "next_eligible_at": next_eligible_at,
+ },
+ )
+ return ActivitySummaryUpdate(status=UpdateStatus.FAILED, error=message, cost_usd=0.0)
+
+
def update_activity_summary(
team_id: int,
user_id: int,
@@ -141,40 +204,43 @@ def update_activity_summary(
if dry_run:
return result
- accounting: dict[str, Any] = {
- "tokens_in": outcome.tokens_in,
- "tokens_out": outcome.tokens_out,
- "cost_usd": cost,
- "latency_ms": outcome.latency_ms,
- "inputs_used": inputs_used,
- "error": outcome.error,
- }
- if outcome.summary is None:
+ try:
+ accounting: dict[str, Any] = {
+ "tokens_in": outcome.tokens_in,
+ "tokens_out": outcome.tokens_out,
+ "cost_usd": cost,
+ "latency_ms": outcome.latency_ms,
+ "inputs_used": inputs_used,
+ "error": outcome.error,
+ }
+ if outcome.summary is None:
+ UserActivitySummary.objects.for_team(team_id).update_or_create(
+ team_id=team_id,
+ user_id=user_id,
+ defaults={
+ **accounting,
+ "status": UserActivitySummary.Status.FAILED,
+ "next_eligible_at": now + RETRY_BACKOFF,
+ },
+ )
+ return result
+
UserActivitySummary.objects.for_team(team_id).update_or_create(
team_id=team_id,
user_id=user_id,
defaults={
**accounting,
- "status": UserActivitySummary.Status.FAILED,
- "next_eligible_at": now + RETRY_BACKOFF,
+ "status": UserActivitySummary.Status.OK,
+ "summary": outcome.summary,
+ "summary_text": summary_text(outcome.summary),
+ "generated_at": now,
+ "model": model,
+ "prompt_version": PROMPT_VERSION,
+ "render_version": RENDER_VERSION,
+ "input_watermark": watermark,
+ "next_eligible_at": now + min_update_interval,
},
)
return result
-
- UserActivitySummary.objects.for_team(team_id).update_or_create(
- team_id=team_id,
- user_id=user_id,
- defaults={
- **accounting,
- "status": UserActivitySummary.Status.OK,
- "summary": outcome.summary,
- "summary_text": summary_text(outcome.summary),
- "generated_at": now,
- "model": model,
- "prompt_version": PROMPT_VERSION,
- "render_version": RENDER_VERSION,
- "input_watermark": watermark,
- "next_eligible_at": now + min_update_interval,
- },
- )
- return result
+ except Exception as error:
+ raise UpdateAfterModelCallError(cost) from error
diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.py
index d637a0675078..4202500f4c7a 100644
--- a/products/signals/backend/activity_summary/queue.py
+++ b/products/signals/backend/activity_summary/queue.py
@@ -15,6 +15,7 @@ from posthog.models import ActivityLog, Team
from products.signals.backend.activity_summary.config import ActivitySummaryConfig
from products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS
+from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIX
from products.signals.backend.activity_summary.prompt import SYSTEM_PROMPT
from products.signals.backend.activity_summary.render import (
MAX_SUMMARY_CHARS,
@@ -76,12 +77,17 @@ def daily_usage(now: datetime.datetime) -> DailyUsage:
One row holds the latest update of a pair, so an earlier update of the same pair in the same 24
hours is not counted. With the minimum update interval that happens only after a failure.
A row without a cost counts as the most one update can cost, so missing usage cannot pass the cap.
+ An update that raised before it called the model is not counted, unless the row holds an earlier paid update.
"""
- # nosemgrep: idor-lookup-without-team (the daily caps are global over all teams, with no user input)
- rows = UserActivitySummary.all_teams.filter(status__in=_CALLED_MODEL, updated_at__gte=now - DAY).aggregate(
- updates=Count("id"),
- unpriced=Count("id", filter=Q(cost_usd__isnull=True)),
- cost=Sum("cost_usd"),
+ rows = (
+ # nosemgrep: idor-lookup-without-team (the daily caps are global over all teams, with no user input)
+ UserActivitySummary.all_teams.filter(status__in=_CALLED_MODEL, updated_at__gte=now - DAY)
+ .exclude(error__startswith=EXCEPTION_ERROR_PREFIX, cost_usd=0)
+ .aggregate(
+ updates=Count("id"),
+ unpriced=Count("id", filter=Q(cost_usd__isnull=True)),
+ cost=Sum("cost_usd"),
+ )
)
return DailyUsage(
updates=rows["updates"],
diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.py
index e6e96826519f..26533a2466b6 100644
--- a/products/signals/backend/activity_summary/tick.py
+++ b/products/signals/backend/activity_summary/tick.py
@@ -8,6 +8,7 @@ import asyncio
import datetime
from collections import Counter
+from django.conf import settings
from django.utils import timezone
import structlog
@@ -25,7 +26,13 @@ from posthog.temporal.common.scoped import scoped_temporal
from posthog.temporal.common.utils import close_db_connections
from products.signals.backend.activity_summary.config import read_config
-from products.signals.backend.activity_summary.logic import ActivitySummaryUpdate, UpdateStatus, update_activity_summary
+from products.signals.backend.activity_summary.logic import (
+ ActivitySummaryUpdate,
+ UpdateAfterModelCallError,
+ UpdateStatus,
+ record_update_exception,
+ update_activity_summary,
+)
from products.signals.backend.activity_summary.prompt import PROMPT_VERSION
from products.signals.backend.activity_summary.queue import (
ActivitySummaryPair,
@@ -54,7 +61,7 @@ _PLAN_TIMEOUT = datetime.timedelta(minutes=5)
# One model call has a 120 second request timeout and one retry on a malformed answer.
_PAIR_TIMEOUT = datetime.timedelta(minutes=5)
_PAIR_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)
-# Only transient errors raise out of the pair activity: `update_activity_summary` records model failures.
+# An attempt that raises is retried. The last attempt records the exception as a failed update instead.
_PAIR_RETRY_POLICY = RetryPolicy(maximum_attempts=2, initial_interval=datetime.timedelta(seconds=10))
@@ -183,6 +190,7 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate
properties={
"team_id": team_id,
"user_id": user_id,
+ "deployment": settings.CLOUD_DEPLOYMENT,
"status": update.status.value,
"error": update.error,
"model": DEFAULT_MODEL,
@@ -198,20 +206,28 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate
)
-def run_pair(input: ActivitySummaryPairInput) -> ActivitySummaryPairResult:
- update = update_activity_summary(
- input.team_id,
- input.user_id,
- min_units=input.min_units,
- min_update_interval=datetime.timedelta(hours=input.min_update_interval_hours),
- )
- called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)
+def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> ActivitySummaryPairResult:
+ try:
+ update = update_activity_summary(
+ input.team_id,
+ input.user_id,
+ min_units=input.min_units,
+ min_update_interval=datetime.timedelta(hours=input.min_update_interval_hours),
+ )
+ called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)
+ except Exception as error:
+ if not last_attempt:
+ raise
+ logger.exception("activity_summary_update_failed", team_id=input.team_id, user_id=input.user_id)
+ # Without a failed row and a backoff, the queue picks the same pair again on the next tick.
+ update = record_update_exception(input.team_id, input.user_id, error)
+ called_model = isinstance(error, UpdateAfterModelCallError)
if update.status == UpdateStatus.OK and input.emit_embeddings:
try:
_emit_embedding(input.team_id, input.user_id)
except Exception:
logger.exception("activity_summary_embedding_failed", team_id=input.team_id, user_id=input.user_id)
- if called_model and input.capture_events:
+ if update.status in (UpdateStatus.OK, UpdateStatus.FAILED) and input.capture_events:
try:
_capture_generated(input.team_id, input.user_id, update)
except Exception:
@@ -233,6 +249,7 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:
distinct_id=PIPELINE_DISTINCT_ID,
event="activity summary tick",
properties={
+ "deployment": settings.CLOUD_DEPLOYMENT,
"candidates": result.candidates,
"picked": result.picked,
**{f"status_{status}": count for status, count in (result.statuses or {}).items()},
@@ -254,8 +271,9 @@ async def plan_activity_summary_tick_activity(input: ActivitySummaryTickInput) -
@scoped_temporal()
@close_db_connections
async def update_activity_summary_pair_activity(input: ActivitySummaryPairInput) -> ActivitySummaryPairResult:
+ last_attempt = activity.info().attempt >= _PAIR_RETRY_POLICY.maximum_attempts
async with Heartbeater():
- return await database_sync_to_async(run_pair, thread_sensitive=False)(input)
+ return await database_sync_to_async(run_pair, thread_sensitive=False)(input, last_attempt=last_attempt)
@activity.defn
diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.py
index ee9717728509..824622eb21d9 100644
--- a/products/signals/backend/test/test_activity_summary_tick.py
+++ b/products/signals/backend/test/test_activity_summary_tick.py
@@ -26,6 +26,7 @@ from products.signals.backend.activity_summary import (
from products.signals.backend.activity_summary.config import ActivitySummaryConfig, parse_config
from products.signals.backend.activity_summary.queue import (
ActivitySummaryPair,
+ DailyUsage,
daily_usage,
max_update_cost_usd,
select_pairs,
@@ -238,8 +239,24 @@ class TestActivitySummaryQueue(BaseTest):
assert (plan.stop_reason, len(plan.pairs), plan.enabled) == (stop_reason, pairs, bool(gateway_key))
- def test_a_skipped_pair_does_not_come_back_on_the_next_tick(self) -> None:
- self._changes(self.user, 2)
+ @parameterized.expand(
+ [
+ ("too few units", 2, {"return_value": FakeLLM([ANSWER])}, False, "skipped_too_few_units", False),
+ (
+ "an exception before the model call",
+ 3,
+ {"side_effect": RuntimeError("no gateway")},
+ False,
+ "failed",
+ False,
+ ),
+ ("an exception after the model call", 3, {"return_value": FakeLLM([ANSWER])}, True, "failed", True),
+ ]
+ )
+ def test_a_pair_that_raised_or_skipped_does_not_come_back_on_the_next_tick(
+ self, _name: str, changes: int, gateway: dict[str, Any], store_fails: bool, status: str, called_model: bool
+ ) -> None:
+ self._changes(self.user, changes)
pair_input = ActivitySummaryPairInput(
team_id=self.team.id,
user_id=self.user.id,
@@ -250,11 +267,49 @@ class TestActivitySummaryQueue(BaseTest):
)
assert self._select() == [(self.team.id, self.user.id)]
- with patch("products.signals.backend.activity_summary.logic.gateway_client", return_value=FakeLLM([ANSWER])):
+ with (
+ patch("products.signals.backend.activity_summary.logic.gateway_client", **gateway),
+ patch(
+ "products.signals.backend.activity_summary.logic.summary_text",
+ side_effect=RuntimeError("store failed") if store_fails else None,
+ return_value="summary",
+ ),
+ ):
result = run_pair(pair_input)
- assert (result.status, result.called_model) == ("skipped_too_few_units", False)
+ assert (result.status, result.called_model) == (status, called_model)
+ assert self._select() == []
+ if called_model:
+ assert result.cost_usd
+ usage = daily_usage(NOW)
+ assert usage.updates == 1
+ assert usage.cost_usd == pytest.approx(result.cost_usd)
+ else:
+ assert daily_usage(NOW) == DailyUsage(updates=0, cost_usd=0.0)
+
+ def test_an_exception_before_the_model_call_keeps_an_earlier_paid_update_in_the_daily_caps(self) -> None:
+ self._changes(self.user, 3)
+ UserActivitySummary.objects.for_team(self.team.id).create(
+ team_id=self.team.id, user_id=self.user.id, status=UserActivitySummary.Status.FAILED, cost_usd=0.25
+ )
+
+ with patch(
+ "products.signals.backend.activity_summary.logic.gateway_client", side_effect=RuntimeError("no gateway")
+ ):
+ result = run_pair(
+ ActivitySummaryPairInput(
+ team_id=self.team.id,
+ user_id=self.user.id,
+ min_units=3,
+ min_update_interval_hours=24,
+ emit_embeddings=True,
+ capture_events=True,
+ )
+ )
+
+ assert (result.status, result.called_model) == ("failed", False)
assert self._select() == []
+ assert daily_usage(NOW) == DailyUsage(updates=1, cost_usd=0.25)
def test_an_ok_update_emits_the_embedding_and_an_event_without_the_summary(self) -> None:
self._changes(self.user, 3)