Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(signals): treat an empty first activity summary profile as a skip (#123888)

posthog[bot]5d9a4aee95c4
diff --git a/products/signals/backend/activity_summary/logic.py b/products/signals/backend/activity_summary/logic.py
index e8d9307efe03..c3b859908d7f 100644
--- a/products/signals/backend/activity_summary/logic.py
+++ b/products/signals/backend/activity_summary/logic.py
@@ -67,6 +67,11 @@ class UpdateStatus(StrEnum):
SKIPPED_NOT_ENROLLED = "skipped_not_enrolled"
SKIPPED_NOT_ELIGIBLE_YET = "skipped_not_eligible_yet"
SKIPPED_TOO_FEW_UNITS = "skipped_too_few_units"
+ SKIPPED_EMPTY_PROFILE = "skipped_empty_profile"
+
+
+# The statuses of an update that called the model, so its cost counts against the daily caps.
+CALLED_MODEL_STATUSES = frozenset({UpdateStatus.OK, UpdateStatus.FAILED, UpdateStatus.SKIPPED_EMPTY_PROFILE})
@frozen
@@ -111,7 +116,7 @@ def is_stale(stored: StoredSummaryVersion | None, rebuild_before: datetime.datet
def skip_backoff(consecutive_skips: int, max_skip_backoff: datetime.timedelta) -> datetime.timedelta:
- """The wait after a skip for too few units, when `consecutive_skips` skips came before it."""
+ """The wait after a skip, when `consecutive_skips` skips came before it."""
return max(RETRY_BACKOFF, min(RETRY_BACKOFF * 2 ** min(consecutive_skips, _MAX_SKIP_DOUBLINGS), max_skip_backoff))
@@ -255,8 +260,9 @@ def update_activity_summary(
run or for a stale summary (see `is_stale`). A stale summary is not sent to the model. A failed
update records the error and keeps the stored summary and the watermark. With `dry_run`, the
model is still called but nothing is written. `force` ignores `next_eligible_at`.
- A failed update sets `next_eligible_at` to `RETRY_BACKOFF` from now. A skip for too few units sets it
- to `skip_backoff` from now, which doubles with each skip until a good update resets it.
+ A failed update sets `next_eligible_at` to `RETRY_BACKOFF` from now. A skip for too few units or for an
+ empty first profile sets it to `skip_backoff` from now, which doubles with each skip until a good update
+ resets it. An empty first profile keeps the cost of its model call.
A successful update also appends a `UserActivitySummaryVersion` and keeps the latest `max_versions`.
"""
team = Team.objects.select_related("organization").get(id=team_id)
@@ -323,8 +329,14 @@ def update_activity_summary(
computed=computed,
)
cost = cost_usd(outcome.tokens_in, outcome.tokens_out, MODEL_PRICES.get(model))
+ if outcome.summary is not None:
+ status = UpdateStatus.OK
+ elif outcome.empty_profile:
+ status = UpdateStatus.SKIPPED_EMPTY_PROFILE
+ else:
+ status = UpdateStatus.FAILED
result = ActivitySummaryUpdate(
- status=UpdateStatus.OK if outcome.summary is not None else UpdateStatus.FAILED,
+ status=status,
rendered_input=rendered.text,
summary=outcome.summary,
tokens_in=outcome.tokens_in,
@@ -350,6 +362,21 @@ def update_activity_summary(
"inputs_used": inputs_used,
"error": outcome.error,
}
+ if outcome.empty_profile:
+ skips = stored.consecutive_skips if stored is not None else 0
+ # `update_or_create` saves `updated_at`, so the daily caps count the model call.
+ UserActivitySummary.objects.for_team(team_id).update_or_create(
+ team_id=team_id,
+ user_id=user_id,
+ defaults={
+ **accounting,
+ "status": UserActivitySummary.Status.EMPTY_PROFILE,
+ "next_eligible_at": now + skip_backoff(skips, max_skip_backoff),
+ "consecutive_skips": min(skips + 1, _MAX_SKIP_DOUBLINGS),
+ },
+ )
+ return result
+
if outcome.summary is None:
UserActivitySummary.objects.for_team(team_id).update_or_create(
team_id=team_id,
diff --git a/products/signals/backend/activity_summary/prompt.py b/products/signals/backend/activity_summary/prompt.py
index c8c842cbd039..dc2aef863cc2 100644
--- a/products/signals/backend/activity_summary/prompt.py
+++ b/products/signals/backend/activity_summary/prompt.py
@@ -122,6 +122,12 @@ class MalformedSummaryError(ValueError):
"""The model's answer cannot be turned into a summary. The message never quotes the answer."""
+class EmptyProfileError(ValueError):
+ """A first summary has a headline but no profile: the input was too thin to describe.
+
+ The same input gives the same answer, so the caller does not retry it."""
+
+
def _profile(value: Any, max_chars: int) -> str | None:
"""The markdown profile with its line breaks, cut at a word boundary when it is longer than `max_chars`."""
if not isinstance(value, str):
@@ -282,7 +288,8 @@ def normalize_summary(
) -> dict[str, Any]:
"""The model's JSON answer under the output rules, or `MalformedSummaryError`.
- Only a missing or empty `profile` or `headline` makes an answer malformed. Every other field
+ Only a missing or empty `profile` or `headline` makes an answer malformed. An empty `profile` with a
+ `headline` and no previous summary raises `EmptyProfileError` instead. Every other field
degrades to an empty or null value, so one bad theme does not cost the row. A theme ref must be
in `computed`, and a theme's `last_seen` is the last day of its refs."""
if not content:
@@ -293,12 +300,15 @@ def normalize_summary(
raise MalformedSummaryError("response is not JSON") from error
if not isinstance(parsed, dict):
raise MalformedSummaryError("response is not a JSON object")
- profile = _profile(parsed.get("profile"), profile_max_chars)
- if profile is None:
- raise MalformedSummaryError("response has no profile")
headline = _clean(parsed.get("headline"))
if headline is None:
raise MalformedSummaryError("response has no headline")
+ profile = _profile(parsed.get("profile"), profile_max_chars)
+ if profile is None:
+ # On an incremental update an empty profile is a regression of the output, not thin input.
+ if had_previous_summary:
+ raise MalformedSummaryError("response has no profile")
+ raise EmptyProfileError("response has no profile")
recent_shift = _clean(parsed.get("recent_shift"), MAX_RECENT_SHIFT_CHARS) if had_previous_summary else None
return {
"profile": profile,
diff --git a/products/signals/backend/activity_summary/queue.py b/products/signals/backend/activity_summary/queue.py
index 313700bf4c86..6e146a0ca165 100644
--- a/products/signals/backend/activity_summary/queue.py
+++ b/products/signals/backend/activity_summary/queue.py
@@ -73,7 +73,11 @@ DAY = datetime.timedelta(days=1)
_BOUND_CHARS_PER_TOKEN = 2
# The cost bound when the model has no price. One update of the default model costs far less.
_UNPRICED_UPDATE_COST_USD = 0.05
-_CALLED_MODEL = (UserActivitySummary.Status.OK, UserActivitySummary.Status.FAILED)
+_CALLED_MODEL = (
+ UserActivitySummary.Status.OK,
+ UserActivitySummary.Status.FAILED,
+ UserActivitySummary.Status.EMPTY_PROFILE,
+)
SOURCE_ACTIVITY_LOG = "activity_log"
SOURCE_TASKS = "tasks"
diff --git a/products/signals/backend/activity_summary/summaries.py b/products/signals/backend/activity_summary/summaries.py
index 0814253df7a2..5b5f4b4a6371 100644
--- a/products/signals/backend/activity_summary/summaries.py
+++ b/products/signals/backend/activity_summary/summaries.py
@@ -16,6 +16,7 @@ from posthog.llm.gateway_client import build_openai_client, trace_id_from_seed
from products.signals.backend.activity_summary.prompt import (
RESPONSE_FORMAT,
+ EmptyProfileError,
MalformedSummaryError,
normalize_summary,
system_prompt,
@@ -64,6 +65,8 @@ class SummaryOutcome:
tokens_out: int | None
latency_ms: int
error: str | None
+ # The model answered a first summary with a headline and no profile. The update stores it as a skip.
+ empty_profile: bool = False
def gateway_client(team_id: int, user_id: int, as_of: datetime.date) -> OpenAI:
@@ -131,6 +134,15 @@ def summarize(
computed=computed,
profile_max_chars=profile_max_chars,
)
+ except EmptyProfileError:
+ return SummaryOutcome(
+ summary=None,
+ tokens_in=tokens_in,
+ tokens_out=tokens_out,
+ latency_ms=round((time.monotonic() - started) * 1000),
+ error=None,
+ empty_profile=True,
+ )
except MalformedSummaryError as malformed:
error = f"malformed_output: {malformed}"
continue
diff --git a/products/signals/backend/activity_summary/tick.py b/products/signals/backend/activity_summary/tick.py
index 724d4d66c687..3a0f8933ff85 100644
--- a/products/signals/backend/activity_summary/tick.py
+++ b/products/signals/backend/activity_summary/tick.py
@@ -28,6 +28,7 @@ from posthog.temporal.common.utils import close_db_connections
from products.signals.backend.activity_summary.config import read_config
from products.signals.backend.activity_summary.logic import (
+ CALLED_MODEL_STATUSES,
MAX_SKIP_BACKOFF,
MAX_VERSIONS,
VIEW_UPDATE_INTERVAL,
@@ -306,7 +307,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A
max_versions=input.max_versions,
max_skip_backoff=datetime.timedelta(hours=input.max_skip_backoff_hours),
)
- called_model = update.status in (UpdateStatus.OK, UpdateStatus.FAILED)
+ called_model = update.status in CALLED_MODEL_STATUSES
except Exception as error:
if not last_attempt:
raise
@@ -324,7 +325,7 @@ def run_pair(input: ActivitySummaryPairInput, *, last_attempt: bool = True) -> A
_emit_embedding(input.team_id, input.user_id)
except Exception:
logger.exception("activity_summary_embedding_failed", team_id=input.team_id, user_id=input.user_id)
- if update.status in (UpdateStatus.OK, UpdateStatus.FAILED) and input.capture_events:
+ if update.status in CALLED_MODEL_STATUSES and input.capture_events:
try:
_capture_generated(input.team_id, input.user_id, update, input.trigger)
except Exception:
diff --git a/products/signals/backend/migrations/0215_useractivitysummary_empty_profile_status.py b/products/signals/backend/migrations/0215_useractivitysummary_empty_profile_status.py
new file mode 100644
index 000000000000..d9d6dbbaf1bf
--- /dev/null
+++ b/products/signals/backend/migrations/0215_useractivitysummary_empty_profile_status.py
@@ -0,0 +1,25 @@
+# Generated by Django 5.2.18 on 2026-10-27 10:09
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+ dependencies = [
+ ("signals", "0214_closed_report_state"),
+ ]
+
+ operations = [
+ migrations.AlterField(
+ model_name="useractivitysummary",
+ name="status",
+ field=models.CharField(
+ choices=[
+ ("ok", "Ok"),
+ ("failed", "Failed"),
+ ("skipped", "Skipped"),
+ ("empty_profile", "Empty Profile"),
+ ],
+ max_length=20,
+ ),
+ ),
+ ]
diff --git a/products/signals/backend/migrations/max_migration.txt b/products/signals/backend/migrations/max_migration.txt
index 2cea72f3c388..8f5f98de0a6f 100644
--- a/products/signals/backend/migrations/max_migration.txt
+++ b/products/signals/backend/migrations/max_migration.txt
@@ -1 +1 @@
-0214_closed_report_state
+0215_useractivitysummary_empty_profile_status
diff --git a/products/signals/backend/models.py b/products/signals/backend/models.py
index 3306a688bab9..c2a53e11b375 100644
--- a/products/signals/backend/models.py
+++ b/products/signals/backend/models.py
@@ -4082,6 +4082,8 @@ class UserActivitySummary(TeamScopedRootMixin, UUIDModel):
FAILED = "failed"
# No update called the model yet. The row only holds the backoff in `next_eligible_at`.
SKIPPED = "skipped"
+ # The model answered a first summary with no profile. The row holds the cost and a skip backoff.
+ EMPTY_PROFILE = "empty_profile"
# See SignalReportRefund.all_teams for rationale.
all_teams = models.Manager() # noqa: DJ012
diff --git a/products/signals/backend/test/test_activity_summary_render.py b/products/signals/backend/test/test_activity_summary_render.py
index 92f76cb3ffa1..614a821f2a8b 100644
--- a/products/signals/backend/test/test_activity_summary_render.py
+++ b/products/signals/backend/test/test_activity_summary_render.py
@@ -13,6 +13,7 @@ from products.signals.backend.activity_summary.computed import compute_fields
from products.signals.backend.activity_summary.prompt import (
MAX_LABEL_CHARS,
MAX_RECENT_SHIFT_CHARS,
+ EmptyProfileError,
MalformedSummaryError,
normalize_summary,
summary_text,
@@ -843,16 +844,20 @@ def test_normalize_summary(_name: str, content: str, had_previous: bool, expecte
@parameterized.expand(
[
- ("empty", None),
- ("not JSON", "Here is the summary: ..."),
- ("not an object", "[1, 2]"),
- ("no profile", _answer(profile=" ")),
- ("no headline", _answer(headline=None)),
+ ("empty", None, False, MalformedSummaryError),
+ ("not JSON", "Here is the summary: ...", False, MalformedSummaryError),
+ ("not an object", "[1, 2]", False, MalformedSummaryError),
+ ("no profile on a first summary", _answer(profile=" "), False, EmptyProfileError),
+ ("no profile with a previous summary", _answer(profile=" "), True, MalformedSummaryError),
+ ("no headline", _answer(headline=None), False, MalformedSummaryError),
+ ("no headline and no profile", _answer(headline=None, profile=""), False, MalformedSummaryError),
]
)
-def test_normalize_summary_rejects_a_malformed_answer(_name: str, content: str | None) -> None:
- with pytest.raises(MalformedSummaryError):
- normalize_summary(content, had_previous_summary=False)
+def test_normalize_summary_rejects_a_malformed_answer(
+ _name: str, content: str | None, had_previous: bool, error: type[Exception]
+) -> None:
+ with pytest.raises(error):
+ normalize_summary(content, had_previous_summary=had_previous)
def test_summary_text_is_the_headline_and_the_theme_labels_in_order() -> None:
@@ -939,6 +944,7 @@ _CONNECTION_ERROR = APIConnectionError(request=SimpleNamespace()) # type: ignor
"api_error: APIConnectionError",
(None, None),
),
+ ("an empty first profile is not retried", [_answer(profile="")], False, 1, None, (100, 20)),
]
)
def test_summarize_retries_a_malformed_or_truncated_answer_once(
@@ -962,6 +968,7 @@ def test_summarize_retries_a_malformed_or_truncated_answer_once(
assert {req["user"] for req in llm.requests} == {PIPELINE_DISTINCT_ID}
assert {req["max_completion_tokens"] for req in llm.requests} == {MAX_COMPLETION_TOKENS}
assert (outcome.summary is not None, outcome.error) == (succeeds, error)
+ assert outcome.empty_profile == (not succeeds and error is None)
assert (outcome.tokens_in, outcome.tokens_out) == tokens
diff --git a/products/signals/backend/test/test_activity_summary_tick.py b/products/signals/backend/test/test_activity_summary_tick.py
index 1c12e4d48569..72f6a0e25844 100644
--- a/products/signals/backend/test/test_activity_summary_tick.py
+++ b/products/signals/backend/test/test_activity_summary_tick.py
@@ -96,6 +96,7 @@ ANSWER = json.dumps(
"recent_shift": None,
}
)
+EMPTY_PROFILE_ANSWER = json.dumps({**json.loads(ANSWER), "profile": "", "themes": [], "modes": []})
_CONFIG_CASES: list[tuple[str, Any, ActivitySummaryConfig]] = [
@@ -644,6 +645,14 @@ class TestActivitySummaryQueue(BaseTest):
False,
),
("an exception after the model call", 3, {"return_value": FakeLLM([ANSWER])}, True, "failed", True),
+ (
+ "an empty first profile",
+ 3,
+ {"return_value": FakeLLM([EMPTY_PROFILE_ANSWER])},
+ False,
+ "skipped_empty_profile",
+ True,
+ ),
]
)
def test_a_pair_that_raised_or_skipped_does_not_come_back_on_the_next_tick(
@@ -947,6 +956,7 @@ async def _run_tick(
(_plan(6, cost_left_usd=3.0), None, "ok", 4, STOP_MAX_COST_USD_PER_DAY, (2, 2)),
# The breaker waits for a sample of 10 model calls, which is five waves.
(_plan(20), 0.01, "failed", 10, STOP_ERROR_RATE, (0, 0)),
+ (_plan(20), 0.01, "skipped_empty_profile", 20, None, (0, 0)),
],
)
async def test_the_tick_bounds_concurrency_and_stops_at_the_daily_caps_and_the_error_rate(
diff --git a/products/signals/backend/test/test_activity_summary_update.py b/products/signals/backend/test/test_activity_summary_update.py
index 96d357a7c03a..1e42a2f31da6 100644
--- a/products/signals/backend/test/test_activity_summary_update.py
+++ b/products/signals/backend/test/test_activity_summary_update.py
@@ -31,6 +31,7 @@ ANSWER = json.dumps(
"recent_shift": None,
}
)
+EMPTY_PROFILE_ANSWER = json.dumps({**json.loads(ANSWER), "profile": "", "themes": [], "modes": []})
@time_machine.travel(NOW, tick=False)
@@ -237,6 +238,54 @@ class TestUpdateActivitySummary(BaseTest):
assert after.version == before.version == 1
assert self._versions() == [1]
+ def test_an_empty_first_profile_is_a_paid_skip_that_backs_off_and_doubles(self) -> None:
+ self._changes(3)
+ llm = FakeLLM([EMPTY_PROFILE_ANSWER])
+
+ result = self._update(llm)
+
+ assert (result.status, result.summary, result.error) == (UpdateStatus.SKIPPED_EMPTY_PROFILE, None, None)
+ assert len(llm.requests) == 1
+ skipped = self._stored()
+ assert skipped is not None
+ assert (skipped.status, skipped.summary, skipped.generated_at, skipped.input_watermark) == (
+ UserActivitySummary.Status.EMPTY_PROFILE,
+ None,
+ None,
+ None,
+ )
+ assert (skipped.tokens_in, skipped.tokens_out, skipped.cost_usd) == (100, 20, result.cost_usd)
+ assert result.cost_usd
+ assert skipped.updated_at == NOW
+ assert skipped.next_eligible_at == NOW + datetime.timedelta(hours=6)
+ assert self._versions() == []
+
+ with time_machine.travel(NOW + datetime.timedelta(hours=7), tick=False):
+ assert self._update(llm).status == UpdateStatus.SKIPPED_EMPTY_PROFILE
+ skipped_again = self._stored()
+ assert skipped_again is not None
+ assert skipped_again.next_eligible_at == NOW + datetime.timedelta(hours=7 + 12)
+
+ def test_an_empty_profile_on_an_incremental_update_fails_and_keeps_the_stored_summary(self) -> None:
+ self._changes(3, days_ago=2)
+ self._update(FakeLLM([ANSWER]))
+ before = self._stored()
+ assert before is not None
+ self._changes(3, days_ago=0.5)
+ llm = FakeLLM([EMPTY_PROFILE_ANSWER])
+
+ result = self._update(llm, force=True)
+
+ assert result.status == UpdateStatus.FAILED
+ assert len(llm.requests) == 2
+ after = self._stored()
+ assert after is not None
+ assert (after.status, after.error) == (
+ UserActivitySummary.Status.FAILED,
+ "malformed_output: response has no profile",
+ )
+ assert (after.summary, after.input_watermark, after.version) == (before.summary, before.input_watermark, 1)
+
def test_keeps_the_latest_max_versions_and_deletes_them_with_the_summary(self) -> None:
for update in range(3):
self._changes(3, days_ago=2 - update / 2)