:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): time each stage of issue summary generation (#123897)
James Gillb16b4ec93277
diff --git a/products/error_tracking/backend/logic/issue_summaries.py b/products/error_tracking/backend/logic/issue_summaries.pyindex e32e0ab72474..cdaa0b7d4ec6 100644--- a/products/error_tracking/backend/logic/issue_summaries.py+++ b/products/error_tracking/backend/logic/issue_summaries.py@@ -8,6 +8,8 @@ import reimport jsonimport timeimport hashlib+from collections.abc import Iterator+from contextlib import contextmanagerfrom datetime import datetime, timedeltafrom enum import StrEnumfrom uuid import UUID, uuid4@@ -19,7 +21,7 @@ import structlogfrom openai import OpenAIfrom openai.types.chat import ChatCompletionMessageParamfrom openai.types.shared_params import ResponseFormatJSONSchema-from prometheus_client import Counter+from prometheus_client import Counter, Histogramfrom pydantic import ValidationErrorfrom redis import Redisfrom redis.exceptions import RedisError@@ -37,8 +39,10 @@ from posthog.exceptions_capture import capture_exceptionfrom posthog.llm.gateway_client import build_openai_client, resolve_ai_gateway_config, team_distinct_idfrom posthog.llm.semantic_enrichment import extract_json_objectfrom posthog.models import Team, User+from posthog.ph_client import ph_background_capturefrom posthog.redis import get_clientfrom posthog.token_bucket import BucketDecision, BucketUnavailable, consume, peek, refund+from posthog.utils import get_instance_regionfrom products.access_control.backend.facade.api import (get_restricted_properties_with_group_type_index_for_team,@@ -104,6 +108,7 @@ logger = structlog.get_logger(__name__)ISSUE_SUMMARY_BUDGET_KEY_PREFIX = "error_tracking_issue_summary"ISSUE_SUMMARY_LOCK_KEY_PREFIX = "error_tracking_issue_summary_lock"ISSUE_SUMMARY_AI_PRODUCT = "error_tracking"+ISSUE_SUMMARY_TASK_EVENT = "error_tracking_issue_summary_generation"# The Go gateway keeps the prompt and the reply out of the AI observability project when this is set.PRIVACY_MODE_HEADER = "X-PostHog-Privacy-Mode"@@ -177,6 +182,36 @@ ISSUE_SUMMARY_OUTCOMES = Counter(labelnames=("outcome",),)+ISSUE_SUMMARY_STAGE_SECONDS = Histogram(+ "error_tracking_issue_summary_stage_seconds",+ "Time spent in each issue summary generation stage",+ labelnames=("stage",),+ buckets=(0.05, 0.1, 0.25, 0.5, 1, 2, 4, 6, 8, 12, 20, 30, 60, 120),+)+++class IssueSummaryTimings:+ """Milliseconds per generation stage, keyed `<stage>_ms`. A stage that did not run has no entry.++ Stages nest, so do not add them up: `total` contains every stage except `queue`, and `inputs` contains+ `latest_event`, `first_event` and `aggregates`.+ """++ def __init__(self) -> None:+ self.ms: dict[str, int] = {}++ @contextmanager+ def stage(self, name: str) -> Iterator[None]:+ started = time.monotonic()+ try:+ yield+ finally:+ self.record(name, time.monotonic() - started)++ def record(self, name: str, seconds: float) -> None:+ self.ms[f"{name}_ms"] = round(seconds * 1000)+ ISSUE_SUMMARY_STAGE_SECONDS.labels(stage=name).observe(seconds)+@frozenclass IssueSummaryResult:@@ -289,7 +324,7 @@ def _publish_generation(team_id: int, issue_id: UUID | str) -> None:transport_options={"socket_timeout": timeout, "socket_connect_timeout": timeout},) as connection:generate_error_tracking_issue_summary.apply_async(- args=(team_id, str(issue_id)), connection=connection, retry=False+ args=(team_id, str(issue_id)), kwargs={"queued_at": time.time()}, connection=connection, retry=False)@@ -312,13 +347,55 @@ def wait_for_issue_summary(time.sleep(min(WAIT_POLL_INTERVAL_SECONDS, remaining))-def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:+def run_issue_summary_task(team_id: int, issue_id: UUID | str, queued_at: float | None) -> None:+ """Generate in the background task, then record how long each stage took.++ `queued_at` is the publisher's wall clock, so the queue time includes any clock skew between the hosts.+ """+ timings = IssueSummaryTimings()+ if queued_at is not None:+ timings.record("queue", max(time.time() - queued_at, 0))+ try:+ result = get_or_generate_issue_summary(team_id, issue_id, timings)+ except Exception:+ _report_task(team_id, issue_id, IssueSummaryOutcome.UNEXPECTED_ERROR, has_summary=False, timings=timings)+ raise+ _report_task(team_id, issue_id, result.outcome, has_summary=result.summary is not None, timings=timings)+++def _report_task(+ team_id: int, issue_id: UUID | str, outcome: IssueSummaryOutcome, *, has_summary: bool, timings: IssueSummaryTimings+) -> None:+ # The background client sends from its own thread, so the worker takes the next task without waiting for a flush.+ try:+ ph_background_capture()(+ distinct_id=team_distinct_id(team_id),+ event=ISSUE_SUMMARY_TASK_EVENT,+ properties={+ "team_id": team_id,+ "issue_id": str(issue_id),+ # The client's `region` super property names the destination project and overwrites an event `region`.+ "instance_region": get_instance_region(),+ "outcome": outcome.value,+ "has_summary": has_summary,+ **timings.ms,+ },+ )+ except Exception:+ logger.warning("error_tracking_issue_summary_capture_failed", team_id=team_id, exc_info=True)+++def get_or_generate_issue_summary(+ team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings | None = None+) -> IssueSummaryResult:"""Return a stored summary, or generate one when none is stored or a stale one is due.Generation runs at most once at a time per issue. A failure keeps the stored summary."""+ timings = timings or IssueSummaryTimings()try:- result = _get_or_generate(team_id, issue_id)+ with timings.stage("total"):+ result = _get_or_generate(team_id, issue_id, timings)except Exception:ISSUE_SUMMARY_OUTCOMES.labels(outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value).inc()raise@@ -329,11 +406,12 @@ def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> IssueSuissue_id=str(issue_id),outcome=result.outcome.value,has_summary=result.summary is not None,+ **timings.ms,)return result-def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:+def _get_or_generate(team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings) -> IssueSummaryResult:stored = get_issue_summary(team_id, issue_id)if (blocked := generation_blocked(team_id, stored)) is not None:return IssueSummaryResult(outcome=blocked, summary=stored)@@ -342,7 +420,7 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:return IssueSummaryResult(outcome=held, summary=stored)try:- result = _generate_claimed(team_id, issue_id)+ result = _generate_claimed(team_id, issue_id, timings)except Exception:_settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR)raise@@ -350,7 +428,7 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:return result-def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:+def _generate_claimed(team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings) -> IssueSummaryResult:# Another worker can finish between the first read and the claim.stored = get_issue_summary(team_id, issue_id)if not regeneration_due(stored, timezone.now()):@@ -364,7 +442,8 @@ def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:team = Team.objects.get(id=team_id)try:- inputs = build_issue_summary_inputs(team, issue_id)+ with timings.stage("inputs"):+ inputs = build_issue_summary_inputs(team, issue_id, timings)except Exception as error:capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)})logger.warning("error_tracking_issue_summary_inputs_failed", team_id=team_id, issue_id=str(issue_id))@@ -390,7 +469,8 @@ def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:return IssueSummaryResult(outcome=IssueSummaryOutcome.GATEWAY_UNCONFIGURED, summary=stored)try:- raw = _call_model(team_id, inputs.prompt)+ with timings.stage("model"):+ raw = _call_model(team_id, issue_id, inputs.prompt)except Exception as error:capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)})logger.warning("error_tracking_issue_summary_model_failed", team_id=team_id, issue_id=str(issue_id))@@ -401,7 +481,8 @@ def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:logger.warning("error_tracking_issue_summary_invalid_output", team_id=team_id, issue_id=str(issue_id))return IssueSummaryResult(outcome=IssueSummaryOutcome.INVALID_OUTPUT, summary=stored)- category = infer_issue_category(team_id, inputs.prompt)+ with timings.stage("category"):+ category = infer_issue_category(team_id, inputs.prompt)generated_at = timezone.now()try:summary, _ = ErrorTrackingIssueSummary.objects.for_team(team_id).update_or_create(@@ -485,7 +566,9 @@ ISSUE_SUMMARY_SOURCE_PROPERTIES = frozenset()-def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummaryInputs | IssueSummaryOutcome:+def build_issue_summary_inputs(+ team: Team, issue_id: UUID | str, timings: IssueSummaryTimings+) -> IssueSummaryInputs | IssueSummaryOutcome:"""Build the prompt and the input hash, or say why the issue cannot be summarised."""issue = ErrorTrackingIssue.objects.with_first_seen().filter(team_id=team.id, id=issue_id).first()if issue is None:@@ -494,12 +577,13 @@ def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummarynow = timezone.now()event_properties: dict[str, object] = {}- for lookback_days in (ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS, ISSUE_SUMMARY_LATEST_EVENT_LOOKBACK_DAYS):- event_properties = _exception_event_properties(- team, issue_id, after=now - timedelta(days=lookback_days), before=now, latest=True- )- if event_properties:- break+ with timings.stage("latest_event"):+ for lookback_days in (ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS, ISSUE_SUMMARY_LATEST_EVENT_LOOKBACK_DAYS):+ event_properties = _exception_event_properties(+ team, issue_id, after=now - timedelta(days=lookback_days), before=now, latest=True+ )+ if event_properties:+ breakstack_trace = (render_stacktrace(mark_in_app_frames(event_properties), ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS)if event_properties@@ -510,15 +594,17 @@ def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummaryfirst_event_properties: dict[str, object] = {}if (window := first_event_window(first_seen)) is not None:- first_event_properties = _exception_event_properties(- team,- issue_id,- after=datetime.fromisoformat(window.after),- before=datetime.fromisoformat(window.before),- latest=False,- )+ with timings.stage("first_event"):+ first_event_properties = _exception_event_properties(+ team,+ issue_id,+ after=datetime.fromisoformat(window.after),+ before=datetime.fromisoformat(window.before),+ latest=False,+ )- aggregated_issue = _aggregated_issue(team, issue_id)+ with timings.stage("aggregates"):+ aggregated_issue = _aggregated_issue(team, issue_id)impact = build_impact(aggregated_issue)trend = build_trend(aggregated_issue, first_seen) or {}latest_release = extract_release(event_properties).get("version")@@ -665,11 +751,11 @@ ISSUE_SUMMARY_RESPONSE_FORMAT: ResponseFormatJSONSchema = {}-def _call_model(team_id: int, prompt: str) -> str:+def _call_model(team_id: int, issue_id: UUID | str, prompt: str) -> str:client: OpenAI = build_openai_client("django",ai_product=ISSUE_SUMMARY_AI_PRODUCT,- properties={"team_id": str(team_id)},+ properties={"team_id": str(team_id), "issue_id": str(issue_id)},distinct_id=team_distinct_id(team_id),)messages: list[ChatCompletionMessageParam] = [diff --git a/products/error_tracking/backend/tasks/tasks.py b/products/error_tracking/backend/tasks/tasks.pyindex 81414b1979c7..6d365136674d 100644--- a/products/error_tracking/backend/tasks/tasks.py+++ b/products/error_tracking/backend/tasks/tasks.py@@ -94,8 +94,8 @@ def dispatch_error_tracking_alert_deliveries(team_id: int, notifications: list[dmax_retries=0,)@with_team_scope()-def generate_error_tracking_issue_summary(team_id: int, issue_id: str) -> None:+def generate_error_tracking_issue_summary(team_id: int, issue_id: str, queued_at: float | None = None) -> None:# The generator imports the LLM client and HogQL, so keep them off every worker's boot path.- from products.error_tracking.backend.logic.issue_summaries import get_or_generate_issue_summary # noqa: PLC0415+ from products.error_tracking.backend.logic.issue_summaries import run_issue_summary_task # noqa: PLC0415- get_or_generate_issue_summary(team_id, issue_id)+ run_issue_summary_task(team_id, issue_id, queued_at)diff --git a/products/error_tracking/backend/tests/test_issue_summaries.py b/products/error_tracking/backend/tests/test_issue_summaries.pyindex ab15ae21d127..b577e7b96fb5 100644--- a/products/error_tracking/backend/tests/test_issue_summaries.py+++ b/products/error_tracking/backend/tests/test_issue_summaries.py@@ -1,6 +1,7 @@import jsonimport signalimport socket+from contextlib import nullcontextfrom datetime import UTC, datetime, timedeltafrom typing import Any@@ -27,6 +28,7 @@ from products.error_tracking.backend.logic import issue_summaries, team_flagsfrom products.error_tracking.backend.logic.issue_summaries import (IssueSummaryInputs,IssueSummaryOutcome,+ IssueSummaryTimings,build_issue_summary_inputs,compute_input_hash,get_or_generate_issue_summary,@@ -130,7 +132,9 @@ class TestIssueSummaryGeneration(BaseTest):self.decide = MagicMock(return_value=_category_result("own_code"))self.enterContext(patch.object(team_flags, "feature_enabled_or_false", return_value=True))self.enterContext(patch.object(issue_summaries, "resolve_ai_gateway_config", return_value=MagicMock()))- self.enterContext(patch.object(issue_summaries, "build_openai_client", return_value=self.client_mock))+ self.build_client = self.enterContext(+ patch.object(issue_summaries, "build_openai_client", return_value=self.client_mock)+ )self.build_inputs = self.enterContext(patch.object(issue_summaries, "build_issue_summary_inputs", side_effect=lambda *_: self.inputs))@@ -179,6 +183,7 @@ class TestIssueSummaryGeneration(BaseTest):assert request["extra_headers"] == {"X-PostHog-Privacy-Mode": "true"}assert request["response_format"]["json_schema"]["strict"] is Trueassert self.decide.call_args.args[0].privacy_mode is True+ assert self.build_client.call_args.kwargs["properties"]["issue_id"] == str(self.issue.id)@parameterized.expand([@@ -193,6 +198,33 @@ class TestIssueSummaryGeneration(BaseTest):assert result.outcome == IssueSummaryOutcome.GENERATED+ @parameterized.expand(+ [+ ("generated", None, IssueSummaryOutcome.GENERATED, {"queue_ms", "total_ms", "inputs_ms", "model_ms"}),+ (+ "unexpected_error",+ OperationalError("db"),+ IssueSummaryOutcome.UNEXPECTED_ERROR,+ {"queue_ms", "total_ms"},+ ),+ ]+ )+ def test_task_reports_outcome_and_stage_timings(+ self, _name: str, error: Exception | None, outcome: IssueSummaryOutcome, stages: set[str]+ ) -> None:+ capture = MagicMock()+ self.enterContext(patch.object(issue_summaries, "ph_background_capture", return_value=capture))+ if error is not None:+ self.enterContext(patch.object(issue_summaries, "get_issue_summary", side_effect=error))++ with time_machine.travel(NOW, tick=False), self.assertRaises(type(error)) if error else nullcontext():+ issue_summaries.run_issue_summary_task(self.team.id, self.issue.id, NOW.timestamp() - 2)++ properties = capture.call_args.kwargs["properties"]+ assert properties["outcome"] == outcome.value+ assert properties["queue_ms"] == 2000+ assert stages <= {key for key in properties if key.endswith("_ms")}+@propertydef _budget_key(self) -> str:return f"{issue_summaries.ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{self.team.id}"@@ -334,6 +366,7 @@ class TestIssueSummaryGeneration(BaseTest):def _start(self) -> tuple[bool, bool]:with patch.object(issue_summaries.generate_error_tracking_issue_summary, "apply_async") as apply_async:started = summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user)+ self.publish_call = apply_async.call_argsreturn started, apply_async.called@parameterized.expand(@@ -353,8 +386,13 @@ class TestIssueSummaryGeneration(BaseTest):if lock_value is not None:issue_summaries.get_client().set(self._lock_key, lock_value, ex=60)- with patch.object(team_flags, "feature_enabled_or_false", return_value=flag):+ with (+ patch.object(team_flags, "feature_enabled_or_false", return_value=flag),+ time_machine.travel(NOW, tick=False),+ ):assert self._start() == (started, queued)+ if queued:+ assert self.publish_call.kwargs["kwargs"] == {"queued_at": NOW.timestamp()}def test_start_generation_queues_when_redis_is_down(self) -> None:with patch.object(issue_summaries, "get_client", side_effect=RedisError("down")):@@ -585,9 +623,11 @@ class TestIssueSummaryInputs(ClickhouseTestMixin, BaseTest):)flush_persons_and_events()- inputs = build_issue_summary_inputs(self.team, issue.id)+ timings = IssueSummaryTimings()+ inputs = build_issue_summary_inputs(self.team, issue.id, timings)assert isinstance(inputs, IssueSummaryInputs)+ assert {"latest_event_ms", "first_event_ms", "aggregates_ms"} <= timings.ms.keys()assert "Exception: TypeError: Cannot read properties of undefined" in inputs.promptassert "\ndispatch in react-dom.js\n[in-app] addToCart in cart.ts line 42" in inputs.promptassert "Handled: no" in inputs.prompt@@ -601,4 +641,6 @@ class TestIssueSummaryInputs(ClickhouseTestMixin, BaseTest):def test_issue_without_exception_events_has_no_inputs(self) -> None:issue = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")- assert build_issue_summary_inputs(self.team, issue.id) == IssueSummaryOutcome.NO_EXCEPTION+ assert (+ build_issue_summary_inputs(self.team, issue.id, IssueSummaryTimings()) == IssueSummaryOutcome.NO_EXCEPTION+ )