:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): back off and record activity summary pairs that raise (#121069)
posthog[bot]ee344c7dc6c2
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.pyindex 61c96115e0f6..108488d6315a 100644--- a/products/signals/backend/activity_summary/logic.py+++ b/products/signals/backend/activity_summary/logic.py@@ -4,6 +4,8 @@ import datetimefrom enum import StrEnumfrom typing import Any+from django.db.models import Value+from django.db.models.functions import Coalescefrom django.utils import timezonefrom posthog.dataclasses import frozen@@ -26,10 +28,21 @@ MIN_UNITS = 3MIN_UPDATE_INTERVAL = datetime.timedelta(days=1)# A pair that skipped or failed waits this long, so it does not take the queue head on every tick.RETRY_BACKOFF = datetime.timedelta(hours=6)+# The error prefixes of an update that raised before it stored a result, before and after the model call.+EXCEPTION_ERROR_PREFIX = "exception: "+EXCEPTION_AFTER_MODEL_CALL_ERROR_PREFIX = "exception_after_model_call: "# The watermark is inclusive in the stored summary, so the next read starts just after it._WATERMARK_STEP = datetime.timedelta(microseconds=1)+class UpdateAfterModelCallError(Exception):+ """An update raised after the model call, so the call counts against the daily caps."""++ def __init__(self, cost_usd: float | None) -> None:+ super().__init__("activity summary update raised after the model call")+ self.cost_usd = cost_usd++class UpdateStatus(StrEnum):OK = "ok"FAILED = "failed"@@ -71,6 +84,56 @@ def _back_off()+def record_update_exception(team_id: int, user_id: int, error: BaseException) -> ActivitySummaryUpdate:+ """Store an update that raised as failed, keep the stored summary and watermark, and back the pair off.++ An update that raised after the model call keeps its cost. A missing cost counts at the bound.+ An update that raised before the model call keeps the cost of the earlier update.+ """+ next_eligible_at = timezone.now() + RETRY_BACKOFF+ if isinstance(error, UpdateAfterModelCallError):+ message = f"{EXCEPTION_AFTER_MODEL_CALL_ERROR_PREFIX}{type(error.__cause__).__name__}"+ UserActivitySummary.objects.for_team(team_id).update_or_create(+ team_id=team_id,+ user_id=user_id,+ defaults={+ "status": UserActivitySummary.Status.FAILED,+ "error": message,+ "tokens_in": None,+ "tokens_out": None,+ "cost_usd": error.cost_usd,+ "latency_ms": None,+ "next_eligible_at": next_eligible_at,+ },+ )+ return ActivitySummaryUpdate(status=UpdateStatus.FAILED, error=message, cost_usd=error.cost_usd)++ message = f"{EXCEPTION_ERROR_PREFIX}{type(error).__name__}"+ # `update()` leaves `updated_at` and a paid cost alone, so the daily caps still count an earlier update.+ updated = (+ UserActivitySummary.objects.for_team(team_id)+ .filter(user_id=user_id)+ .update(+ status=UserActivitySummary.Status.FAILED,+ error=message,+ cost_usd=Coalesce("cost_usd", Value(0.0)),+ next_eligible_at=next_eligible_at,+ )+ )+ if not updated:+ UserActivitySummary.objects.for_team(team_id).get_or_create(+ team_id=team_id,+ user_id=user_id,+ defaults={+ "status": UserActivitySummary.Status.FAILED,+ "error": message,+ "cost_usd": 0.0,+ "next_eligible_at": next_eligible_at,+ },+ )+ return ActivitySummaryUpdate(status=UpdateStatus.FAILED, error=message, cost_usd=0.0)++def update_activity_summary(team_id: int,user_id: int,@@ -141,40 +204,43 @@ def update_activity_summary(if dry_run:return result- accounting: dict[str, Any] = {- "tokens_in": outcome.tokens_in,- "tokens_out": outcome.tokens_out,- "cost_usd": cost,- "latency_ms": outcome.latency_ms,- "inputs_used": inputs_used,- "error": outcome.error,- }- if outcome.summary is None:+ try:+ accounting: dict[str, Any] = {+ "tokens_in": outcome.tokens_in,+ "tokens_out": outcome.tokens_out,+ "cost_usd": cost,+ "latency_ms": outcome.latency_ms,+ "inputs_used": inputs_used,+ "error": outcome.error,+ }+ if outcome.summary is None:+ UserActivitySummary.objects.for_team(team_id).update_or_create(+ team_id=team_id,+ user_id=user_id,+ defaults={+ **accounting,+ "status": UserActivitySummary.Status.FAILED,+ "next_eligible_at": now + RETRY_BACKOFF,+ },+ )+ return result+UserActivitySummary.objects.for_team(team_id).update_or_create(team_id=team_id,user_id=user_id,defaults={**accounting,- "status": UserActivitySummary.Status.FAILED,- "next_eligible_at": now + RETRY_BACKOFF,+ "status": UserActivitySummary.Status.OK,+ "summary": outcome.summary,+ "summary_text": summary_text(outcome.summary),+ "generated_at": now,+ "model": model,+ "prompt_version": PROMPT_VERSION,+ "render_version": RENDER_VERSION,+ "input_watermark": watermark,+ "next_eligible_at": now + min_update_interval,},)return result-- UserActivitySummary.objects.for_team(team_id).update_or_create(- team_id=team_id,- user_id=user_id,- defaults={- **accounting,- "status": UserActivitySummary.Status.OK,- "summary": outcome.summary,- "summary_text": summary_text(outcome.summary),- "generated_at": now,- "model": model,- "prompt_version": PROMPT_VERSION,- "render_version": RENDER_VERSION,- "input_watermark": watermark,- "next_eligible_at": now + min_update_interval,- },- )- return result+ except Exception as error:+ raise UpdateAfterModelCallError(cost) from errordiff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.pyindex d637a0675078..4202500f4c7a 100644--- a/products/signals/backend/activity_summary/queue.py+++ b/products/signals/backend/activity_summary/queue.py@@ -15,6 +15,7 @@ from posthog.models import ActivityLog, Teamfrom products.signals.backend.activity_summary.config import ActivitySummaryConfigfrom products.signals.backend.activity_summary.inputs import TASK_ORIGIN_PRODUCTS+from products.signals.backend.activity_summary.logic import EXCEPTION_ERROR_PREFIXfrom products.signals.backend.activity_summary.prompt import SYSTEM_PROMPTfrom products.signals.backend.activity_summary.render import (MAX_SUMMARY_CHARS,@@ -76,12 +77,17 @@ def daily_usage(now: datetime.datetime) -> DailyUsage:One row holds the latest update of a pair, so an earlier update of the same pair in the same 24hours is not counted. With the minimum update interval that happens only after a failure.A row without a cost counts as the most one update can cost, so missing usage cannot pass the cap.+ An update that raised before it called the model is not counted, unless the row holds an earlier paid update."""- # nosemgrep: idor-lookup-without-team (the daily caps are global over all teams, with no user input)- rows = UserActivitySummary.all_teams.filter(status__in=_CALLED_MODEL, updated_at__gte=now - DAY).aggregate(- updates=Count("id"),- unpriced=Count("id", filter=Q(cost_usd__isnull=True)),- cost=Sum("cost_usd"),+ rows = (+ # nosemgrep: idor-lookup-without-team (the daily caps are global over all teams, with no user input)+ UserActivitySummary.all_teams.filter(status__in=_CALLED_MODEL, updated_at__gte=now - DAY)+ .exclude(error__startswith=EXCEPTION_ERROR_PREFIX, cost_usd=0)+ .aggregate(+ updates=Count("id"),+ unpriced=Count("id", filter=Q(cost_usd__isnull=True)),+ cost=Sum("cost_usd"),+ ))return DailyUsage(updates=rows["updates"],diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.pyindex e6e96826519f..26533a2466b6 100644--- a/products/signals/backend/activity_summary/tick.py+++ b/products/signals/backend/activity_summary/tick.py@@ -8,6 +8,7 @@ import asyncioimport datetimefrom collections import Counter+from django.conf import settingsfrom django.utils import timezoneimport structlog@@ -25,7 +26,13 @@ from posthog.temporal.common.scoped import scoped_temporalfrom posthog.temporal.common.utils import close_db_connectionsfrom products.signals.backend.activity_summary.config import read_config-from products.signals.backend.activity_summary.logic import ActivitySummaryUpdate, UpdateStatus, update_activity_summary+from products.signals.backend.activity_summary.logic import (+ ActivitySummaryUpdate,+ UpdateAfterModelCallError,+ UpdateStatus,+ record_update_exception,+ update_activity_summary,+)from products.signals.backend.activity_summary.prompt import PROMPT_VERSIONfrom products.signals.backend.activity_summary.queue import (ActivitySummaryPair,@@ -54,7 +61,7 @@ _PLAN_TIMEOUT = datetime.timedelta(minutes=5)# One model call has a 120 second request timeout and one retry on a malformed answer._PAIR_TIMEOUT = datetime.timedelta(minutes=5)_PAIR_HEARTBEAT_TIMEOUT = datetime.timedelta(minutes=1)-# Only transient errors raise out of the pair activity: `update_activity_summary` records model failures.+# An attempt that raises is retried. The last attempt records the exception as a failed update instead._PAIR_RETRY_POLICY = RetryPolicy(maximum_attempts=2, initial_interval=datetime.timedelta(seconds=10))@@ -183,6 +190,7 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdateproperties={"team_id": team_id,"user_id": user_id,+ "deployment": settings.CLOUD_DEPLOYMENT,"status": update.status.value,"error": update.error,"model": DEFAULT_MODEL,@@ -198,20 +206,28 @@ def _capture_generated(team_id: int, user_id: int, update: ActivitySummaryUpdate)-def run_pair(input: ActivitySummaryPairInput) -> ActivitySummaryPairResult:- update = update_activity_summary(- input.team_id,- input.user_id,- min_units=input.min_units,- min_update_interval=datetime.timedelta(hours=input.min_update_interval_hours),- )- called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)+def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> ActivitySummaryPairResult:+ try:+ update = update_activity_summary(+ input.team_id,+ input.user_id,+ min_units=input.min_units,+ min_update_interval=datetime.timedelta(hours=input.min_update_interval_hours),+ )+ called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)+ except Exception as error:+ if not last_attempt:+ raise+ logger.exception("activity_summary_update_failed", team_id=input.team_id, user_id=input.user_id)+ # Without a failed row and a backoff, the queue picks the same pair again on the next tick.+ update = record_update_exception(input.team_id, input.user_id, error)+ called_model = isinstance(error, UpdateAfterModelCallError)if update.status == UpdateStatus.OK and input.emit_embeddings:try:_emit_embedding(input.team_id, input.user_id)except Exception:logger.exception("activity_summary_embedding_failed", team_id=input.team_id, user_id=input.user_id)- if called_model and input.capture_events:+ if update.status in (UpdateStatus.OK, UpdateStatus.FAILED) and input.capture_events:try:_capture_generated(input.team_id, input.user_id, update)except Exception:@@ -233,6 +249,7 @@ def capture_tick(result: ActivitySummaryTickResult) -> None:distinct_id=PIPELINE_DISTINCT_ID,event="activity summary tick",properties={+ "deployment": settings.CLOUD_DEPLOYMENT,"candidates": result.candidates,"picked": result.picked,**{f"status_{status}": count for status, count in (result.statuses or {}).items()},@@ -254,8 +271,9 @@ async def plan_activity_summary_tick_activity(input: ActivitySummaryTickInput) -@scoped_temporal()@close_db_connectionsasync def update_activity_summary_pair_activity(input: ActivitySummaryPairInput) -> ActivitySummaryPairResult:+ last_attempt = activity.info().attempt >= _PAIR_RETRY_POLICY.maximum_attemptsasync with Heartbeater():- return await database_sync_to_async(run_pair, thread_sensitive=False)(input)+ return await database_sync_to_async(run_pair, thread_sensitive=False)(input, last_attempt=last_attempt)@activity.defndiff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.pyindex ee9717728509..824622eb21d9 100644--- a/products/signals/backend/test/test_activity_summary_tick.py+++ b/products/signals/backend/test/test_activity_summary_tick.py@@ -26,6 +26,7 @@ from products.signals.backend.activity_summary import (from products.signals.backend.activity_summary.config import ActivitySummaryConfig, parse_configfrom products.signals.backend.activity_summary.queue import (ActivitySummaryPair,+ DailyUsage,daily_usage,max_update_cost_usd,select_pairs,@@ -238,8 +239,24 @@ class TestActivitySummaryQueue(BaseTest):assert (plan.stop_reason, len(plan.pairs), plan.enabled) == (stop_reason, pairs, bool(gateway_key))- def test_a_skipped_pair_does_not_come_back_on_the_next_tick(self) -> None:- self._changes(self.user, 2)+ @parameterized.expand(+ [+ ("too few units", 2, {"return_value": FakeLLM([ANSWER])}, False, "skipped_too_few_units", False),+ (+ "an exception before the model call",+ 3,+ {"side_effect": RuntimeError("no gateway")},+ False,+ "failed",+ False,+ ),+ ("an exception after the model call", 3, {"return_value": FakeLLM([ANSWER])}, True, "failed", True),+ ]+ )+ def test_a_pair_that_raised_or_skipped_does_not_come_back_on_the_next_tick(+ self, _name: str, changes: int, gateway: dict[str, Any], store_fails: bool, status: str, called_model: bool+ ) -> None:+ self._changes(self.user, changes)pair_input = ActivitySummaryPairInput(team_id=self.team.id,user_id=self.user.id,@@ -250,11 +267,49 @@ class TestActivitySummaryQueue(BaseTest):)assert self._select() == [(self.team.id, self.user.id)]- with patch("products.signals.backend.activity_summary.logic.gateway_client", return_value=FakeLLM([ANSWER])):+ with (+ patch("products.signals.backend.activity_summary.logic.gateway_client", **gateway),+ patch(+ "products.signals.backend.activity_summary.logic.summary_text",+ side_effect=RuntimeError("store failed") if store_fails else None,+ return_value="summary",+ ),+ ):result = run_pair(pair_input)- assert (result.status, result.called_model) == ("skipped_too_few_units", False)+ assert (result.status, result.called_model) == (status, called_model)+ assert self._select() == []+ if called_model:+ assert result.cost_usd+ usage = daily_usage(NOW)+ assert usage.updates == 1+ assert usage.cost_usd == pytest.approx(result.cost_usd)+ else:+ assert daily_usage(NOW) == DailyUsage(updates=0, cost_usd=0.0)++ def test_an_exception_before_the_model_call_keeps_an_earlier_paid_update_in_the_daily_caps(self) -> None:+ self._changes(self.user, 3)+ UserActivitySummary.objects.for_team(self.team.id).create(+ team_id=self.team.id, user_id=self.user.id, status=UserActivitySummary.Status.FAILED, cost_usd=0.25+ )++ with patch(+ "products.signals.backend.activity_summary.logic.gateway_client", side_effect=RuntimeError("no gateway")+ ):+ result = run_pair(+ ActivitySummaryPairInput(+ team_id=self.team.id,+ user_id=self.user.id,+ min_units=3,+ min_update_interval_hours=24,+ emit_embeddings=True,+ capture_events=True,+ )+ )++ assert (result.status, result.called_model) == ("failed", False)assert self._select() == []+ assert daily_usage(NOW) == DailyUsage(updates=1, cost_usd=0.25)def test_an_ok_update_emits_the_embedding_and_an_event_without_the_summary(self) -> None:self._changes(self.user, 3)