:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(signals): treat an empty first activity summary profile as a skip (#123888)
posthog[bot]5d9a4aee95c4
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.pyindex e8d9307efe03..c3b859908d7f 100644--- a/products/signals/backend/activity_summary/logic.py+++ b/products/signals/backend/activity_summary/logic.py@@ -67,6 +67,11 @@ class UpdateStatus(StrEnum):SKIPPED_NOT_ENROLLED = "skipped_not_enrolled"SKIPPED_NOT_ELIGIBLE_YET = "skipped_not_eligible_yet"SKIPPED_TOO_FEW_UNITS = "skipped_too_few_units"+ SKIPPED_EMPTY_PROFILE = "skipped_empty_profile"+++# The statuses of an update that called the model, so its cost counts against the daily caps.+CALLED_MODEL_STATUSES = frozenset({UpdateStatus.OK, UpdateStatus.FAILED, UpdateStatus.SKIPPED_EMPTY_PROFILE})@frozen@@ -111,7 +116,7 @@ def is_stale(stored: StoredSummaryVersion | None, rebuild_before: datetime.datetdef skip_backoff(consecutive_skips: int, max_skip_backoff: datetime.timedelta) -> datetime.timedelta:- """The wait after a skip for too few units, when `consecutive_skips` skips came before it."""+ """The wait after a skip, when `consecutive_skips` skips came before it."""return max(RETRY_BACKOFF, min(RETRY_BACKOFF * 2 ** min(consecutive_skips, _MAX_SKIP_DOUBLINGS), max_skip_backoff))@@ -255,8 +260,9 @@ def update_activity_summary(run or for a stale summary (see `is_stale`). A stale summary is not sent to the model. A failedupdate records the error and keeps the stored summary and the watermark. With `dry_run`, themodel is still called but nothing is written. `force` ignores `next_eligible_at`.- A failed update sets `next_eligible_at` to `RETRY_BACKOFF` from now. A skip for too few units sets it- to `skip_backoff` from now, which doubles with each skip until a good update resets it.+ A failed update sets `next_eligible_at` to `RETRY_BACKOFF` from now. A skip for too few units or for an+ empty first profile sets it to `skip_backoff` from now, which doubles with each skip until a good update+ resets it. An empty first profile keeps the cost of its model call.A successful update also appends a `UserActivitySummaryVersion` and keeps the latest `max_versions`."""team = Team.objects.select_related("organization").get(id=team_id)@@ -323,8 +329,14 @@ def update_activity_summary(computed=computed,)cost = cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(model))+ if outcome.summary is not None:+ status = UpdateStatus.OK+ elif outcome.empty_profile:+ status = UpdateStatus.SKIPPED_EMPTY_PROFILE+ else:+ status = UpdateStatus.FAILEDresult = ActivitySummaryUpdate(- status=UpdateStatus.OK if outcome.summary is not None else UpdateStatus.FAILED,+ status=status,rendered_input=rendered.text,summary=outcome.summary,tokens_in=outcome.tokens_in,@@ -350,6 +362,21 @@ def update_activity_summary("inputs_used": inputs_used,"error": outcome.error,}+ if outcome.empty_profile:+ skips = stored.consecutive_skips if stored is not None else 0+ # `update_or_create` saves `updated_at`, so the daily caps count the model call.+ UserActivitySummary.objects.for_team(team_id).update_or_create(+ team_id=team_id,+ user_id=user_id,+ defaults={+ **accounting,+ "status": UserActivitySummary.Status.EMPTY_PROFILE,+ "next_eligible_at": now + skip_backoff(skips, max_skip_backoff),+ "consecutive_skips": min(skips + 1, _MAX_SKIP_DOUBLINGS),+ },+ )+ return result+if outcome.summary is None:UserActivitySummary.objects.for_team(team_id).update_or_create(team_id=team_id,diff --git a/products/signals/backend/activity_summary/prompt.py b/products/signals/backend/activity_summary/prompt.pyindex c8c842cbd039..dc2aef863cc2 100644--- a/products/signals/backend/activity_summary/prompt.py+++ b/products/signals/backend/activity_summary/prompt.py@@ -122,6 +122,12 @@ class MalformedSummaryError(ValueError):"""The model's answer cannot be turned into a summary. The message never quotes the answer."""+class EmptyProfileError(ValueError):+ """A first summary has a headline but no profile: the input was too thin to describe.++ The same input gives the same answer, so the caller does not retry it."""++def _profile(value: Any, max_chars: int) -> str | None:"""The markdown profile with its line breaks, cut at a word boundary when it is longer than `max_chars`."""if not isinstance(value, str):@@ -282,7 +288,8 @@ def normalize_summary() -> dict[str, Any]:"""The model's JSON answer under the output rules, or `MalformedSummaryError`.- Only a missing or empty `profile` or `headline` makes an answer malformed. Every other field+ Only a missing or empty `profile` or `headline` makes an answer malformed. An empty `profile` with a+ `headline` and no previous summary raises `EmptyProfileError` instead. Every other fielddegrades to an empty or null value, so one bad theme does not cost the row. A theme ref must bein `computed`, and a theme's `last_seen` is the last day of its refs."""if not content:@@ -293,12 +300,15 @@ def normalize_summary(raise MalformedSummaryError("response is not JSON") from errorif not isinstance(parsed, dict):raise MalformedSummaryError("response is not a JSON object")- profile = _profile(parsed.get("profile"), profile_max_chars)- if profile is None:- raise MalformedSummaryError("response has no profile")headline = _clean(parsed.get("headline"))if headline is None:raise MalformedSummaryError("response has no headline")+ profile = _profile(parsed.get("profile"), profile_max_chars)+ if profile is None:+ # On an incremental update an empty profile is a regression of the output, not thin input.+ if had_previous_summary:+ raise MalformedSummaryError("response has no profile")+ raise EmptyProfileError("response has no profile")recent_shift = _clean(parsed.get("recent_shift"), MAX_RECENT_SHIFT_CHARS) if had_previous_summary else Nonereturn {"profile": profile,diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.pyindex 313700bf4c86..6e146a0ca165 100644--- a/products/signals/backend/activity_summary/queue.py+++ b/products/signals/backend/activity_summary/queue.py@@ -73,7 +73,11 @@ DAY = datetime.timedelta(days=1)_BOUND_CHARS_PER_TOKEN = 2# The cost bound when the model has no price. One update of the default model costs far less._UNPRICED_UPDATE_COST_USD = 0.05-_CALLED_MODEL = (UserActivitySummary.Status.OK, UserActivitySummary.Status.FAILED)+_CALLED_MODEL = (+ UserActivitySummary.Status.OK,+ UserActivitySummary.Status.FAILED,+ UserActivitySummary.Status.EMPTY_PROFILE,+)SOURCE_ACTIVITY_LOG = "activity_log"SOURCE_TASKS = "tasks"diff --git a/products/signals/backend/activity_summary/summaries.py b/products/signals/backend/activity_summary/summaries.pyindex 0814253df7a2..5b5f4b4a6371 100644--- a/products/signals/backend/activity_summary/summaries.py+++ b/products/signals/backend/activity_summary/summaries.py@@ -16,6 +16,7 @@ from posthog.llm.gateway_client import build_openai_client, trace_id_from_seedfrom products.signals.backend.activity_summary.prompt import (RESPONSE_FORMAT,+ EmptyProfileError,MalformedSummaryError,normalize_summary,system_prompt,@@ -64,6 +65,8 @@ class SummaryOutcome:tokens_out: int | Nonelatency_ms: interror: str | None+ # The model answered a first summary with a headline and no profile. The update stores it as a skip.+ empty_profile: bool = Falsedef gateway_client(team_id: int, user_id: int, as_of: datetime.date) -> OpenAI:@@ -131,6 +134,15 @@ def summarize(computed=computed,profile_max_chars=profile_max_chars,)+ except EmptyProfileError:+ return SummaryOutcome(+ summary=None,+ tokens_in=tokens_in,+ tokens_out=tokens_out,+ latency_ms=round((time.monotonic() - started) * 1000),+ error=None,+ empty_profile=True,+ )except MalformedSummaryError as malformed:error = f"malformed_output: {malformed}"continuediff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.pyindex 724d4d66c687..3a0f8933ff85 100644--- a/products/signals/backend/activity_summary/tick.py+++ b/products/signals/backend/activity_summary/tick.py@@ -28,6 +28,7 @@ from posthog.temporal.common.utils import close_db_connectionsfrom products.signals.backend.activity_summary.config import read_configfrom products.signals.backend.activity_summary.logic import (+ CALLED_MODEL_STATUSES,MAX_SKIP_BACKOFF,MAX_VERSIONS,VIEW_UPDATE_INTERVAL,@@ -306,7 +307,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> Amax_versions=input.max_versions,max_skip_backoff=datetime.timedelta(hours=input.max_skip_backoff_hours),)- called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)+ called_model = update.status in CALLED_MODEL_STATUSESexcept Exception as error:if not last_attempt:raise@@ -324,7 +325,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A_emit_embedding(input.team_id, input.user_id)except Exception:logger.exception("activity_summary_embedding_failed", team_id=input.team_id, user_id=input.user_id)- if update.status in (UpdateStatus.OK, UpdateStatus.FAILED) and input.capture_events:+ if update.status in CALLED_MODEL_STATUSES and input.capture_events:try:_capture_generated(input.team_id, input.user_id, update, input.trigger)except Exception:diff --git a/products/signals/backend/migrations/0215_useractivitysummary_empty_profile_status.py b/products/signals/backend/migrations/0215_useractivitysummary_empty_profile_status.pynew file mode 100644index 000000000000..d9d6dbbaf1bf--- /dev/null+++ b/products/signals/backend/migrations/0215_useractivitysummary_empty_profile_status.py@@ -0,0 +1,25 @@+# Generated by Django 5.2.18 on 2026-10-27 10:09++from django.db import migrations, models+++class Migration(migrations.Migration):+ dependencies = [+ ("signals", "0214_closed_report_state"),+ ]++ operations = [+ migrations.AlterField(+ model_name="useractivitysummary",+ name="status",+ field=models.CharField(+ choices=[+ ("ok", "Ok"),+ ("failed", "Failed"),+ ("skipped", "Skipped"),+ ("empty_profile", "Empty Profile"),+ ],+ max_length=20,+ ),+ ),+ ]diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txtindex 2cea72f3c388..8f5f98de0a6f 100644--- a/products/signals/backend/migrations/max_migration.txt+++ b/products/signals/backend/migrations/max_migration.txt@@ -1 +1 @@-0214_closed_report_state+0215_useractivitysummary_empty_profile_statusdiff --git a/products/signals/backend/models.py b/products/signals/backend/models.pyindex 3306a688bab9..c2a53e11b375 100644--- a/products/signals/backend/models.py+++ b/products/signals/backend/models.py@@ -4082,6 +4082,8 @@ class UserActivitySummary(TeamScopedRootMixin, UUIDModel):FAILED = "failed"# No update called the model yet. The row only holds the backoff in `next_eligible_at`.SKIPPED = "skipped"+ # The model answered a first summary with no profile. The row holds the cost and a skip backoff.+ EMPTY_PROFILE = "empty_profile"# See SignalReportRefund.all_teams for rationale.all_teams = models.Manager() # noqa: DJ012diff --git a/products/signals/backend/test/test_activity_summary_render.py b/products/signals/backend/test/test_activity_summary_render.pyindex 92f76cb3ffa1..614a821f2a8b 100644--- a/products/signals/backend/test/test_activity_summary_render.py+++ b/products/signals/backend/test/test_activity_summary_render.py@@ -13,6 +13,7 @@ from products.signals.backend.activity_summary.computed import compute_fieldsfrom products.signals.backend.activity_summary.prompt import (MAX_LABEL_CHARS,MAX_RECENT_SHIFT_CHARS,+ EmptyProfileError,MalformedSummaryError,normalize_summary,summary_text,@@ -843,16 +844,20 @@ def test_normalize_summary(_name: str, content: str, had_previous: bool, expecte@parameterized.expand([- ("empty", None),- ("not JSON", "Here is the summary: ..."),- ("not an object", "[1, 2]"),- ("no profile", _answer(profile=" ")),- ("no headline", _answer(headline=None)),+ ("empty", None, False, MalformedSummaryError),+ ("not JSON", "Here is the summary: ...", False, MalformedSummaryError),+ ("not an object", "[1, 2]", False, MalformedSummaryError),+ ("no profile on a first summary", _answer(profile=" "), False, EmptyProfileError),+ ("no profile with a previous summary", _answer(profile=" "), True, MalformedSummaryError),+ ("no headline", _answer(headline=None), False, MalformedSummaryError),+ ("no headline and no profile", _answer(headline=None, profile=""), False, MalformedSummaryError),])-def test_normalize_summary_rejects_a_malformed_answer(_name: str, content: str | None) -> None:- with pytest.raises(MalformedSummaryError):- normalize_summary(content, had_previous_summary=False)+def test_normalize_summary_rejects_a_malformed_answer(+ _name: str, content: str | None, had_previous: bool, error: type[Exception]+) -> None:+ with pytest.raises(error):+ normalize_summary(content, had_previous_summary=had_previous)def test_summary_text_is_the_headline_and_the_theme_labels_in_order() -> None:@@ -939,6 +944,7 @@ _CONNECTION_ERROR = APIConnectionError(request=SimpleNamespace()) # type: ignor"api_error: APIConnectionError",(None, None),),+ ("an empty first profile is not retried", [_answer(profile="")], False, 1, None, (100, 20)),])def test_summarize_retries_a_malformed_or_truncated_answer_once(@@ -962,6 +968,7 @@ def test_summarize_retries_a_malformed_or_truncated_answer_once(assert {req["user"] for req in llm.requests} == {PIPELINE_DISTINCT_ID}assert {req["max_completion_tokens"] for req in llm.requests} == {MAX_COMPLETION_TOKENS}assert (outcome.summary is not None, outcome.error) == (succeeds, error)+ assert outcome.empty_profile == (not succeeds and error is None)assert (outcome.tokens_in, outcome.tokens_out) == tokensdiff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.pyindex 1c12e4d48569..72f6a0e25844 100644--- a/products/signals/backend/test/test_activity_summary_tick.py+++ b/products/signals/backend/test/test_activity_summary_tick.py@@ -96,6 +96,7 @@ ANSWER = json.dumps("recent_shift": None,})+EMPTY_PROFILE_ANSWER = json.dumps({**json.loads(ANSWER), "profile": "", "themes": [], "modes": []})_CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [@@ -644,6 +645,14 @@ class TestActivitySummaryQueue(BaseTest):False,),("an exception after the model call", 3, {"return_value": FakeLLM([ANSWER])}, True, "failed", True),+ (+ "an empty first profile",+ 3,+ {"return_value": FakeLLM([EMPTY_PROFILE_ANSWER])},+ False,+ "skipped_empty_profile",+ True,+ ),])def test_a_pair_that_raised_or_skipped_does_not_come_back_on_the_next_tick(@@ -947,6 +956,7 @@ async def _run_tick((_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY, (2, 2)),# The breaker waits for a sample of 10 model calls, which is five waves.(_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE, (0, 0)),+ (_plan(20), 0.01, "skipped_empty_profile", 20, None, (0, 0)),],)async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps_and_the_error_rate(diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.pyindex 96d357a7c03a..1e42a2f31da6 100644--- a/products/signals/backend/test/test_activity_summary_update.py+++ b/products/signals/backend/test/test_activity_summary_update.py@@ -31,6 +31,7 @@ ANSWER = json.dumps("recent_shift": None,})+EMPTY_PROFILE_ANSWER = json.dumps({**json.loads(ANSWER), "profile": "", "themes": [], "modes": []})@time_machine.travel(NOW, tick=False)@@ -237,6 +238,54 @@ class TestUpdateActivitySummary(BaseTest):assert after.version == before.version == 1assert self._versions() == [1]+ def test_an_empty_first_profile_is_a_paid_skip_that_backs_off_and_doubles(self) -> None:+ self._changes(3)+ llm = FakeLLM([EMPTY_PROFILE_ANSWER])++ result = self._update(llm)++ assert (result.status, result.summary, result.error) == (UpdateStatus.SKIPPED_EMPTY_PROFILE, None, None)+ assert len(llm.requests) == 1+ skipped = self._stored()+ assert skipped is not None+ assert (skipped.status, skipped.summary, skipped.generated_at, skipped.input_watermark) == (+ UserActivitySummary.Status.EMPTY_PROFILE,+ None,+ None,+ None,+ )+ assert (skipped.tokens_in, skipped.tokens_out, skipped.cost_usd) == (100, 20, result.cost_usd)+ assert result.cost_usd+ assert skipped.updated_at == NOW+ assert skipped.next_eligible_at == NOW + datetime.timedelta(hours=6)+ assert self._versions() == []++ with time_machine.travel(NOW + datetime.timedelta(hours=7), tick=False):+ assert self._update(llm).status == UpdateStatus.SKIPPED_EMPTY_PROFILE+ skipped_again = self._stored()+ assert skipped_again is not None+ assert skipped_again.next_eligible_at == NOW + datetime.timedelta(hours=7 + 12)++ def test_an_empty_profile_on_an_incremental_update_fails_and_keeps_the_stored_summary(self) -> None:+ self._changes(3, days_ago=2)+ self._update(FakeLLM([ANSWER]))+ before = self._stored()+ assert before is not None+ self._changes(3, days_ago=0.5)+ llm = FakeLLM([EMPTY_PROFILE_ANSWER])++ result = self._update(llm, force=True)++ assert result.status == UpdateStatus.FAILED+ assert len(llm.requests) == 2+ after = self._stored()+ assert after is not None+ assert (after.status, after.error) == (+ UserActivitySummary.Status.FAILED,+ "malformed_output: response has no profile",+ )+ assert (after.summary, after.input_watermark, after.version) == (before.summary, before.input_watermark, 1)+def test_keeps_the_latest_max_versions_and_deletes_them_with_the_summary(self) -> None:for update in range(3):self._changes(3, days_ago=2 - update / 2)