:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): generate an issue summary from the event in hand (#124158)
James Gill29cb8c014664
diff --git a/products/error_tracking/backend/logic/issue_summaries.py b/products/error_tracking/backend/logic/issue_summaries.pyindex cdaa0b7d4ec6..e9e1298ea893 100644--- a/products/error_tracking/backend/logic/issue_summaries.py+++ b/products/error_tracking/backend/logic/issue_summaries.py@@ -1,14 +1,15 @@"""Generate, store and read the model-written summary of an issue.-Every trigger calls `get_or_generate_issue_summary`, directly or through the Celery task. It returns-the stored summary when no new one is due, so callers need no freshness logic of their own.+A read calls `get_or_generate_issue_summary`, directly or through the Celery task. The issue-created workflow+calls `generate_issue_summary_from_event`. Both return the stored summary when no new one is due, so callers+need no freshness logic of their own."""import reimport jsonimport timeimport hashlib-from collections.abc import Iterator+from collections.abc import Callable, Iteratorfrom contextlib import contextmanagerfrom datetime import datetime, timedeltafrom enum import StrEnum@@ -41,7 +42,7 @@ from posthog.llm.semantic_enrichment import extract_json_objectfrom posthog.models import Team, Userfrom posthog.ph_client import ph_background_capturefrom posthog.redis import get_client-from posthog.token_bucket import BucketDecision, BucketUnavailable, consume, peek, refund+from posthog.token_bucket import BucketDecision, BucketUnavailable, Budget, consume, peek, refundfrom posthog.utils import get_instance_regionfrom products.access_control.backend.facade.api import (@@ -257,10 +258,20 @@ def regeneration_due(summary: ErrorTrackingIssueSummary | None, now: datetime) -return summary is None or now - summary.checked_at >= ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL-def generation_blocked(team_id: int, stored: ErrorTrackingIssueSummary | None) -> IssueSummaryOutcome | None:+def first_summary_due(summary: ErrorTrackingIssueSummary | None, _now: datetime) -> bool:+ # Generation on issue creation sees only the first event, so any stored summary already covers it.+ return summary is None+++DueCheck = Callable[[ErrorTrackingIssueSummary | None, datetime], bool]+++def generation_blocked(+ team_id: int, stored: ErrorTrackingIssueSummary | None, due: DueCheck+) -> IssueSummaryOutcome | None:"""The reason no generation can start now, or None when one can."""# Checked first because it needs no network call or query, and most reads find a fresh summary.- if not regeneration_due(stored, timezone.now()):+ if not due(stored, timezone.now()):return IssueSummaryOutcome.FRESHif not issue_summaries_enabled(team_id):return IssueSummaryOutcome.DISABLED@@ -274,7 +285,7 @@ def start_issue_summary_generation(team_id: int, issue_id: UUID | str) -> bool:False means nothing will change soon, so the caller should not wait."""- if generation_blocked(team_id, get_issue_summary(team_id, issue_id)) is not None:+ if generation_blocked(team_id, get_issue_summary(team_id, issue_id), regeneration_due) is not None:return Falsetry:client = _start_client()@@ -355,12 +366,38 @@ def run_issue_summary_task(team_id: int, issue_id: UUID | str, queued_at: floattimings = IssueSummaryTimings()if queued_at is not None:timings.record("queue", max(time.time() - queued_at, 0))+ _generate_and_report(team_id, issue_id, timings, lambda: get_or_generate_issue_summary(team_id, issue_id, timings))+++def generate_issue_summary_from_event(+ team_id: int, issue_id: UUID | str, event_properties: dict[str, object], *, budget_key: str, budget: Budget+) -> IssueSummaryResult:+ """Generate the first summary of a new issue from its first event, with no ClickHouse query.++ Does nothing when a summary is already stored. A stored summary refreshes through the read path.+ """+ generation = _SummaryGeneration(+ due=first_summary_due,+ budget_key=budget_key,+ budget=budget,+ build_inputs=lambda team, _timings: build_issue_summary_inputs_from_event(team, issue_id, event_properties),+ )+ timings = IssueSummaryTimings()+ return _generate_and_report(+ team_id, issue_id, timings, lambda: _get_or_generate_counted(team_id, issue_id, generation, timings)+ )+++def _generate_and_report(+ team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings, generate: Callable[[], IssueSummaryResult]+) -> IssueSummaryResult:try:- result = get_or_generate_issue_summary(team_id, issue_id, timings)+ result = generate()except Exception:_report_task(team_id, issue_id, IssueSummaryOutcome.UNEXPECTED_ERROR, has_summary=False, timings=timings)raise_report_task(team_id, issue_id, result.outcome, has_summary=result.summary is not None, timings=timings)+ return resultdef _report_task(@@ -392,10 +429,31 @@ def get_or_generate_issue_summary(Generation runs at most once at a time per issue. A failure keeps the stored summary."""- timings = timings or IssueSummaryTimings()+ generation = _SummaryGeneration(+ due=regeneration_due,+ budget_key=f"{ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{team_id}",+ budget=ISSUE_SUMMARY_BUDGET,+ build_inputs=lambda team, inputs_timings: build_issue_summary_inputs(team, issue_id, inputs_timings),+ )+ return _get_or_generate_counted(team_id, issue_id, generation, timings or IssueSummaryTimings())+++@frozen+class _SummaryGeneration:+ """What differs between a generation on read and one on issue creation."""++ due: DueCheck+ budget_key: str+ budget: Budget+ build_inputs: Callable[[Team, IssueSummaryTimings], IssueSummaryInputs | IssueSummaryOutcome]+++def _get_or_generate_counted(+ team_id: int, issue_id: UUID | str, generation: _SummaryGeneration, timings: IssueSummaryTimings+) -> IssueSummaryResult:try:with timings.stage("total"):- result = _get_or_generate(team_id, issue_id, timings)+ result = _get_or_generate(team_id, issue_id, generation, timings)except Exception:ISSUE_SUMMARY_OUTCOMES.labels(outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value).inc()raise@@ -411,16 +469,18 @@ def get_or_generate_issue_summary(return result-def _get_or_generate(team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings) -> IssueSummaryResult:+def _get_or_generate(+ team_id: int, issue_id: UUID | str, generation: _SummaryGeneration, timings: IssueSummaryTimings+) -> IssueSummaryResult:stored = get_issue_summary(team_id, issue_id)- if (blocked := generation_blocked(team_id, stored)) is not None:+ if (blocked := generation_blocked(team_id, stored, generation.due)) is not None:return IssueSummaryResult(outcome=blocked, summary=stored)claim = f"{LOCK_CLAIM_PREFIX}{uuid4().hex}"if (held := _claim_generation(team_id, issue_id, claim)) is not None:return IssueSummaryResult(outcome=held, summary=stored)try:- result = _generate_claimed(team_id, issue_id, timings)+ result = _generate_claimed(team_id, issue_id, generation, timings)except Exception:_settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR)raise@@ -428,22 +488,24 @@ def _get_or_generate(team_id: int, issue_id: UUID | str, timings: IssueSummaryTireturn result-def _generate_claimed(team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings) -> IssueSummaryResult:+def _generate_claimed(+ team_id: int, issue_id: UUID | str, generation: _SummaryGeneration, timings: IssueSummaryTimings+) -> IssueSummaryResult:# Another worker can finish between the first read and the claim.stored = get_issue_summary(team_id, issue_id)- if not regeneration_due(stored, timezone.now()):+ if not generation.due(stored, timezone.now()):return IssueSummaryResult(outcome=IssueSummaryOutcome.FRESH, summary=stored)- budget_key = f"{ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{team_id}"+ budget_key, budget = generation.budget_key, generation.budget# The input queries run before `consume`, so an unchanged issue spends no token. This read keeps a team# with no tokens left from running them for every issue it opens. `consume` still makes the decision.- if _budget_exhausted(budget_key):+ if _budget_exhausted(budget_key, budget):return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)team = Team.objects.get(id=team_id)try:with timings.stage("inputs"):- inputs = build_issue_summary_inputs(team, issue_id, timings)+ inputs = generation.build_inputs(team, timings)except Exception as error:capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)})logger.warning("error_tracking_issue_summary_inputs_failed", team_id=team_id, issue_id=str(issue_id))@@ -455,7 +517,7 @@ def _generate_claimed(team_id: int, issue_id: UUID | str, timings: IssueSummaryTErrorTrackingIssueSummary.objects.for_team(team_id).filter(id=stored.id).update(checked_at=stored.checked_at)return IssueSummaryResult(outcome=IssueSummaryOutcome.UNCHANGED, summary=stored)- decision = consume(budget_key, ISSUE_SUMMARY_BUDGET)+ decision = consume(budget_key, budget)match decision:case BucketDecision(allowed=False):return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)@@ -465,7 +527,7 @@ def _generate_claimed(team_id: int, issue_id: UUID | str, timings: IssueSummaryT# Only the Go gateway honours privacy mode, so the Python gateway fallback is never used.if resolve_ai_gateway_config() is None:- refund(budget_key, ISSUE_SUMMARY_BUDGET)+ refund(budget_key, budget)return IssueSummaryResult(outcome=IssueSummaryOutcome.GATEWAY_UNCONFIGURED, summary=stored)try:@@ -502,8 +564,8 @@ def _generate_claimed(team_id: int, issue_id: UUID | str, timings: IssueSummaryTreturn IssueSummaryResult(outcome=IssueSummaryOutcome.GENERATED, summary=summary)-def _budget_exhausted(budget_key: str) -> bool:- decision = peek(budget_key, ISSUE_SUMMARY_BUDGET)+def _budget_exhausted(budget_key: str, budget: Budget) -> bool:+ decision = peek(budget_key, budget)return isinstance(decision, BucketDecision) and not decision.allowed@@ -560,7 +622,7 @@ def compute_input_hash(*, fingerprints: list[str], latest_release: str | None, o# The only event properties the prompt builder sees, and the ones `summary_sources_readable` checks.-# `_exception_event_properties` drops every other key, so a property left out is missing from the prompt, not leaked.+# `_source_properties` drops every other key, so a property left out is missing from the prompt, not leaked.ISSUE_SUMMARY_SOURCE_PROPERTIES = frozenset({"$exception_list", "$exception_level", "$lib", "$current_url", "$exception_release"})@@ -570,7 +632,7 @@ def build_issue_summary_inputs(team: Team, issue_id: UUID | str, timings: IssueSummaryTimings) -> IssueSummaryInputs | IssueSummaryOutcome:"""Build the prompt and the input hash, or say why the issue cannot be summarised."""- issue = ErrorTrackingIssue.objects.with_first_seen().filter(team_id=team.id, id=issue_id).first()+ issue = _issue_with_first_seen(team, issue_id)if issue is None:return IssueSummaryOutcome.ISSUE_MISSINGfirst_seen: datetime | None = getattr(issue, "first_seen", None)@@ -584,11 +646,7 @@ def build_issue_summary_inputs()if event_properties:break- stack_trace = (- render_stacktrace(mark_in_app_frames(event_properties), ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS)- if event_properties- else ""- )+ stack_trace = _summary_stack_trace(event_properties)if not stack_trace:return IssueSummaryOutcome.NO_EXCEPTION@@ -605,8 +663,63 @@ def build_issue_summary_inputs(with timings.stage("aggregates"):aggregated_issue = _aggregated_issue(team, issue_id)- impact = build_impact(aggregated_issue)- trend = build_trend(aggregated_issue, first_seen) or {}+ return _summary_inputs(+ team,+ issue_id,+ first_seen=first_seen,+ event_properties=event_properties,+ first_event_properties=first_event_properties,+ stack_trace=stack_trace,+ impact=build_impact(aggregated_issue),+ trend=build_trend(aggregated_issue, first_seen) or {},+ )+++def build_issue_summary_inputs_from_event(+ team: Team, issue_id: UUID | str, event_properties: dict[str, object]+) -> IssueSummaryInputs | IssueSummaryOutcome:+ """Build the inputs of a new issue from its only event, which is both its first and its latest."""+ issue = _issue_with_first_seen(team, issue_id)+ if issue is None:+ return IssueSummaryOutcome.ISSUE_MISSING+ source_properties = _source_properties(event_properties)+ stack_trace = _summary_stack_trace(source_properties)+ if not stack_trace:+ return IssueSummaryOutcome.NO_EXCEPTION+ session_id = event_properties.get("$session_id")+ return _summary_inputs(+ team,+ issue_id,+ first_seen=getattr(issue, "first_seen", None),+ event_properties=source_properties,+ first_event_properties=source_properties,+ stack_trace=stack_trace,+ impact={"occurrences": 1, "users": 1, "sessions": 1 if isinstance(session_id, str) and session_id else 0},+ trend={},+ )+++def _issue_with_first_seen(team: Team, issue_id: UUID | str) -> ErrorTrackingIssue | None:+ return ErrorTrackingIssue.objects.with_first_seen().filter(team_id=team.id, id=issue_id).first()+++def _summary_stack_trace(event_properties: dict[str, object]) -> str:+ if not event_properties:+ return ""+ return render_stacktrace(mark_in_app_frames(event_properties), ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS)+++def _summary_inputs(+ team: Team,+ issue_id: UUID | str,+ *,+ first_seen: datetime | None,+ event_properties: dict[str, object],+ first_event_properties: dict[str, object],+ stack_trace: str,+ impact: dict[str, object],+ trend: dict[str, object],+) -> IssueSummaryInputs:latest_release = extract_release(event_properties).get("version")first_release = extract_release(first_event_properties).get("version")exception = first_exception(event_properties)@@ -717,7 +830,10 @@ def _exception_event_properties()if not response.results:return {}- properties = as_record(parse_jsonish(response.results[0][0])) or {}+ return _source_properties(as_record(parse_jsonish(response.results[0][0])) or {})+++def _source_properties(properties: dict[str, object]) -> dict[str, object]:return {key: value for key, value in properties.items() if key in ISSUE_SUMMARY_SOURCE_PROPERTIES}diff --git a/products/error_tracking/backend/tests/test_issue_summaries.py b/products/error_tracking/backend/tests/test_issue_summaries.pyindex b577e7b96fb5..1ec5e6f03564 100644--- a/products/error_tracking/backend/tests/test_issue_summaries.py+++ b/products/error_tracking/backend/tests/test_issue_summaries.py@@ -30,7 +30,9 @@ from products.error_tracking.backend.logic.issue_summaries import (IssueSummaryOutcome,IssueSummaryTimings,build_issue_summary_inputs,+ build_issue_summary_inputs_from_event,compute_input_hash,+ generate_issue_summary_from_event,get_or_generate_issue_summary,parse_summary_response,)@@ -138,6 +140,9 @@ class TestIssueSummaryGeneration(BaseTest):self.build_inputs = self.enterContext(patch.object(issue_summaries, "build_issue_summary_inputs", side_effect=lambda *_: self.inputs))+ self.enterContext(+ patch.object(issue_summaries, "build_issue_summary_inputs_from_event", side_effect=lambda *_: self.inputs)+ )self.enterContext(patch.object(issue_summaries.ml_inference, "decide_when_available", self.decide))def tearDown(self) -> None:@@ -350,6 +355,48 @@ class TestIssueSummaryGeneration(BaseTest):assert shown is not Noneassert (shown.generated_at, shown.checked_at) == (result.summary.generated_at, result.summary.checked_at)+ def _generate_on_creation(+ self, budget_key: str | None = None, budget: Budget = issue_summaries.ISSUE_SUMMARY_BUDGET+ ) -> IssueSummaryOutcome:+ return generate_issue_summary_from_event(+ self.team.id,+ self.issue.id,+ {},+ budget_key=budget_key or self._budget_key,+ budget=budget,+ ).outcome++ def test_generation_on_creation_runs_once_and_keeps_a_stored_summary(self) -> None:+ with time_machine.travel(NOW, tick=False):+ assert self._generate_on_creation() == IssueSummaryOutcome.GENERATED+ self._release_lock()+ self.inputs = IssueSummaryInputs(prompt="Exception: TypeError", input_hash="hash-2")++ with time_machine.travel(NOW + timedelta(hours=25), tick=False):+ assert self._generate_on_creation() == IssueSummaryOutcome.FRESH+ assert self.llm_calls == 1+ stored = issue_summaries.get_issue_summary(self.team.id, self.issue.id)+ assert stored is not None and stored.input_hash == "hash-1"++ def test_generation_on_creation_reports_outcome_and_stage_timings(self) -> None:+ capture = MagicMock()+ self.enterContext(patch.object(issue_summaries, "ph_background_capture", return_value=capture))++ assert self._generate_on_creation() == IssueSummaryOutcome.GENERATED++ assert capture.call_args.kwargs["event"] == issue_summaries.ISSUE_SUMMARY_TASK_EVENT+ properties = capture.call_args.kwargs["properties"]+ assert properties["outcome"] == IssueSummaryOutcome.GENERATED.value+ assert {"total_ms", "inputs_ms", "model_ms"} <= properties.keys()++ def test_generation_on_creation_spends_the_budget_its_caller_passes(self) -> None:+ creation_key = f"{self._budget_key}:on-creation"+ tiny = Budget(burst=1, per_hour=1)+ consume(creation_key, tiny)++ assert self._generate_on_creation(budget_key=creation_key, budget=tiny) == IssueSummaryOutcome.RATE_LIMITED+ assert self.llm_calls == 0+def test_merge_keeps_target_summary_and_deletes_source_summary(self) -> None:source = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")ErrorTrackingIssueFingerprintV2.objects.create(team=self.team, issue=source, fingerprint="fp-source")@@ -638,9 +685,55 @@ class TestIssueSummaryInputs(ClickhouseTestMixin, BaseTest):assert inputs.prompt.count("</issue>") == 1assert inputs.input_hash == compute_input_hash(fingerprints=["fp-1"], latest_release="1.2.0", occurrences=2)+ @time_machine.travel(NOW, tick=False)+ def test_event_in_hand_matches_clickhouse_hash_for_a_one_event_issue(self) -> None:+ issue = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")+ ErrorTrackingIssueFingerprintV2.objects.create(team=self.team, issue=issue, fingerprint="fp-1")+ sync_issues_to_clickhouse(issue_ids=[issue.id], team_id=self.team.pk)+ properties: dict[str, object] = {+ "$exception_fingerprint": "fp-1",+ "$exception_issue_id": str(issue.id),+ "$exception_release": {"version": "2.0.0"},+ "$session_id": "01890000-0000-7000-8000-000000000001",+ "$exception_list": [+ {+ "type": "TypeError",+ "value": "Cannot read properties of undefined",+ "stacktrace": {+ "type": "resolved",+ "frames": [{"resolved_name": "addToCart", "source": "cart.ts", "line": 42, "in_app": True}],+ },+ }+ ],+ }+ _create_event(+ distinct_id="user-1",+ event="$exception",+ team=self.team,+ timestamp=NOW - timedelta(minutes=5),+ properties=properties,+ )+ flush_persons_and_events()++ from_clickhouse = build_issue_summary_inputs(self.team, issue.id, IssueSummaryTimings())+ from_event = build_issue_summary_inputs_from_event(self.team, issue.id, properties)++ assert isinstance(from_clickhouse, IssueSummaryInputs)+ assert isinstance(from_event, IssueSummaryInputs)+ assert from_event.input_hash == from_clickhouse.input_hash+ assert "Impact in range: 1 occurrences, 1 users, 1 sessions" in from_event.prompt++ without_session = {key: value for key, value in properties.items() if key != "$session_id"}+ from_sessionless_event = build_issue_summary_inputs_from_event(self.team, issue.id, without_session)+ assert isinstance(from_sessionless_event, IssueSummaryInputs)+ assert "Impact in range: 1 occurrences, 1 users, 0 sessions" in from_sessionless_event.prompt+def test_issue_without_exception_events_has_no_inputs(self) -> None:issue = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")assert (build_issue_summary_inputs(self.team, issue.id, IssueSummaryTimings()) == IssueSummaryOutcome.NO_EXCEPTION)+ assert build_issue_summary_inputs_from_event(self.team, issue.id, {"$lib": "web"}) == (+ IssueSummaryOutcome.NO_EXCEPTION+ )