:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(error-tracking): generate the first issue summary when the issue is created (#124778)
James Gillc22329b75d42
diff --git a/products/error_tracking/backend/logic/issue_summaries.py b/products/error_tracking/backend/logic/issue_summaries.pyindex e9e1298ea893..5345a66a8e1c 100644--- a/products/error_tracking/backend/logic/issue_summaries.py+++ b/products/error_tracking/backend/logic/issue_summaries.py@@ -75,10 +75,12 @@ from products.error_tracking.backend.logic.issue_summary_config import (ISSUE_SUMMARY_MAX_OUTPUT_TOKENS,ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL,ISSUE_SUMMARY_MODEL,+ ISSUE_SUMMARY_ON_WRITE_FLAG,ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,ISSUE_SUMMARY_QUEUED_TTL_SECONDS,ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS,ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS,+ ISSUE_SUMMARY_WRITE_BUDGET,)from products.error_tracking.backend.logic.issue_summary_prompt import (ISSUE_SUMMARY_CATEGORY_QUESTION,@@ -107,6 +109,7 @@ from products.ml_inference.backend.facade.contracts import ChoiceAnswer, Decisiologger = structlog.get_logger(__name__)ISSUE_SUMMARY_BUDGET_KEY_PREFIX = "error_tracking_issue_summary"+ISSUE_SUMMARY_WRITE_BUDGET_KEY_PREFIX = "error_tracking_issue_summary_write"ISSUE_SUMMARY_LOCK_KEY_PREFIX = "error_tracking_issue_summary_lock"ISSUE_SUMMARY_AI_PRODUCT = "error_tracking"ISSUE_SUMMARY_TASK_EVENT = "error_tracking_issue_summary_generation"@@ -144,8 +147,8 @@ elseendreturn 1"""-# Deletes the key only while it holds the queued marker, so a worker's claim or a hold outcome survives.-RELEASE_QUEUED_LUA = """+# Deletes the key only while it holds ARGV[1], so a value another caller wrote survives.+RELEASE_LUA = """if redis.call('GET', KEYS[1]) == ARGV[1] thenreturn redis.call('DEL', KEYS[1])end@@ -153,6 +156,11 @@ return 0"""+class IssueSummaryTrigger(StrEnum):+ READ = "read"+ WRITE = "write"++class IssueSummaryOutcome(StrEnum):GENERATED = "generated"# A stored summary is younger than the regenerate interval.@@ -179,8 +187,8 @@ SETTLED_OUTCOMES = frozenset({IssueSummaryOutcome.GENERATED, IssueSummaryOutcomeISSUE_SUMMARY_OUTCOMES = Counter("error_tracking_issue_summary_total",- "Issue summary requests by outcome",- labelnames=("outcome",),+ "Issue summary requests by outcome and trigger",+ labelnames=("outcome", "trigger"),)ISSUE_SUMMARY_STAGE_SECONDS = Histogram(@@ -231,6 +239,10 @@ def issue_summaries_enabled(team_id: int) -> bool:return team_flag_enabled(ISSUE_SUMMARY_FLAG, team_id)+def issue_summaries_on_write_enabled(team_id: int) -> bool:+ return team_flag_enabled(ISSUE_SUMMARY_ON_WRITE_FLAG, team_id)++def summary_sources_readable(team_id: int, user: User) -> bool:"""False when the user may not read an event property the summary is written from.@@ -306,7 +318,7 @@ def _queue_generation(team_id: int, issue_id: UUID | str) -> bool:capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)})# Without a task, the queued marker would keep reads waiting until it expires.try:- _start_client().register_script(RELEASE_QUEUED_LUA)(keys=[_lock_key(team_id, issue_id)], args=[LOCK_QUEUED])+ _start_client().register_script(RELEASE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[LOCK_QUEUED])except RedisError:logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)return False@@ -366,42 +378,68 @@ def run_issue_summary_task(team_id: int, issue_id: UUID | str, queued_at: floattimings = IssueSummaryTimings()if queued_at is not None:timings.record("queue", max(time.time() - queued_at, 0))- _generate_and_report(team_id, issue_id, timings, lambda: get_or_generate_issue_summary(team_id, issue_id, timings))+ _generate_and_report(+ team_id,+ issue_id,+ IssueSummaryTrigger.READ,+ timings,+ lambda: get_or_generate_issue_summary(team_id, issue_id, timings),+ )def generate_issue_summary_from_event(- team_id: int, issue_id: UUID | str, event_properties: dict[str, object], *, budget_key: str, budget: Budget+ team_id: int, issue_id: UUID | str, load_event_properties: Callable[[Team], dict[str, object]]) -> IssueSummaryResult:"""Generate the first summary of a new issue from its first event, with no ClickHouse query.Does nothing when a summary is already stored. A stored summary refreshes through the read path.+ `load_event_properties` runs only after every check passes, so a blocked attempt reads no event."""generation = _SummaryGeneration(+ trigger=IssueSummaryTrigger.WRITE,due=first_summary_due,- budget_key=budget_key,- budget=budget,- build_inputs=lambda team, _timings: build_issue_summary_inputs_from_event(team, issue_id, event_properties),+ budget_key=f"{ISSUE_SUMMARY_WRITE_BUDGET_KEY_PREFIX}:{team_id}",+ budget=ISSUE_SUMMARY_WRITE_BUDGET,+ build_inputs=lambda team, _timings: build_issue_summary_inputs_from_event(+ team, issue_id, load_event_properties(team)+ ),)timings = IssueSummaryTimings()return _generate_and_report(- team_id, issue_id, timings, lambda: _get_or_generate_counted(team_id, issue_id, generation, timings)+ team_id,+ issue_id,+ generation.trigger,+ timings,+ lambda: _get_or_generate_counted(team_id, issue_id, generation, timings),)def _generate_and_report(- team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings, generate: Callable[[], IssueSummaryResult]+ team_id: int,+ issue_id: UUID | str,+ trigger: IssueSummaryTrigger,+ timings: IssueSummaryTimings,+ generate: Callable[[], IssueSummaryResult],) -> IssueSummaryResult:try:result = generate()except Exception:- _report_task(team_id, issue_id, IssueSummaryOutcome.UNEXPECTED_ERROR, has_summary=False, timings=timings)+ _report_task(+ team_id, issue_id, trigger, IssueSummaryOutcome.UNEXPECTED_ERROR, has_summary=False, timings=timings+ )raise- _report_task(team_id, issue_id, result.outcome, has_summary=result.summary is not None, timings=timings)+ _report_task(team_id, issue_id, trigger, result.outcome, has_summary=result.summary is not None, timings=timings)return resultdef _report_task(- team_id: int, issue_id: UUID | str, outcome: IssueSummaryOutcome, *, has_summary: bool, timings: IssueSummaryTimings+ team_id: int,+ issue_id: UUID | str,+ trigger: IssueSummaryTrigger,+ outcome: IssueSummaryOutcome,+ *,+ has_summary: bool,+ timings: IssueSummaryTimings,) -> None:# The background client sends from its own thread, so the worker takes the next task without waiting for a flush.try:@@ -413,6 +451,7 @@ def _report_task("issue_id": str(issue_id),# The client's `region` super property names the destination project and overwrites an event `region`."instance_region": get_instance_region(),+ "trigger": trigger.value,"outcome": outcome.value,"has_summary": has_summary,**timings.ms,@@ -430,6 +469,7 @@ def get_or_generate_issue_summary(Generation runs at most once at a time per issue. A failure keeps the stored summary."""generation = _SummaryGeneration(+ trigger=IssueSummaryTrigger.READ,due=regeneration_due,budget_key=f"{ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{team_id}",budget=ISSUE_SUMMARY_BUDGET,@@ -442,11 +482,16 @@ def get_or_generate_issue_summary(class _SummaryGeneration:"""What differs between a generation on read and one on issue creation."""+ trigger: IssueSummaryTriggerdue: DueCheckbudget_key: strbudget: Budgetbuild_inputs: Callable[[Team, IssueSummaryTimings], IssueSummaryInputs | IssueSummaryOutcome]+ @property+ def fails_closed(self) -> bool:+ return self.trigger == IssueSummaryTrigger.WRITE+def _get_or_generate_counted(team_id: int, issue_id: UUID | str, generation: _SummaryGeneration, timings: IssueSummaryTimings@@ -455,14 +500,17 @@ def _get_or_generate_counted(with timings.stage("total"):result = _get_or_generate(team_id, issue_id, generation, timings)except Exception:- ISSUE_SUMMARY_OUTCOMES.labels(outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value).inc()+ ISSUE_SUMMARY_OUTCOMES.labels(+ outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value, trigger=generation.trigger.value+ ).inc()raise- ISSUE_SUMMARY_OUTCOMES.labels(outcome=result.outcome.value).inc()+ ISSUE_SUMMARY_OUTCOMES.labels(outcome=result.outcome.value, trigger=generation.trigger.value).inc()logger.info("error_tracking_issue_summary",team_id=team_id,issue_id=str(issue_id),outcome=result.outcome.value,+ trigger=generation.trigger.value,has_summary=result.summary is not None,**timings.ms,)@@ -476,15 +524,18 @@ def _get_or_generate(if (blocked := generation_blocked(team_id, stored, generation.due)) is not None:return IssueSummaryResult(outcome=blocked, summary=stored)claim = f"{LOCK_CLAIM_PREFIX}{uuid4().hex}"- if (held := _claim_generation(team_id, issue_id, claim)) is not None:+ if (held := _claim_generation(team_id, issue_id, claim, fails_closed=generation.fails_closed)) is not None:return IssueSummaryResult(outcome=held, summary=stored)+ # A write attempt that leaves no summary deletes its claim instead of writing its outcome to the lock.+ # The next read then generates at once, so the write budget or a write failure never blocks a read.+ release = generation.trigger == IssueSummaryTrigger.WRITEtry:result = _generate_claimed(team_id, issue_id, generation, timings)except Exception:- _settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR)+ _settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR, release=release)raise- _settle_claim(team_id, issue_id, claim, result.outcome)+ _settle_claim(team_id, issue_id, claim, result.outcome, release=release)return result@@ -499,7 +550,7 @@ def _generate_claimed(budget_key, budget = generation.budget_key, generation.budget# The input queries run before `consume`, so an unchanged issue spends no token. This read keeps a team# with no tokens left from running them for every issue it opens. `consume` still makes the decision.- if _budget_exhausted(budget_key, budget):+ if _budget_exhausted(budget_key, budget, fails_closed=generation.fails_closed):return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)team = Team.objects.get(id=team_id)@@ -522,8 +573,10 @@ def _generate_claimed(case BucketDecision(allowed=False):return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)case BucketUnavailable(error=bucket_error):- # Fail open: the per-issue lock and the regenerate interval still bound the spend.logger.warning("error_tracking_issue_summary_budget_unavailable", team_id=team_id, error=bucket_error)+ if generation.fails_closed:+ return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)+ # A read fails open: the per-issue lock and the regenerate interval still bound the spend.# Only the Go gateway honours privacy mode, so the Python gateway fallback is never used.if resolve_ai_gateway_config() is None:@@ -564,16 +617,20 @@ def _generate_claimed(return IssueSummaryResult(outcome=IssueSummaryOutcome.GENERATED, summary=summary)-def _budget_exhausted(budget_key: str, budget: Budget) -> bool:+def _budget_exhausted(budget_key: str, budget: Budget, *, fails_closed: bool) -> bool:decision = peek(budget_key, budget)- return isinstance(decision, BucketDecision) and not decision.allowed+ if isinstance(decision, BucketUnavailable):+ return fails_closed+ return not decision.alloweddef _lock_key(team_id: int, issue_id: UUID | str) -> str:return f"{ISSUE_SUMMARY_LOCK_KEY_PREFIX}:{team_id}:{issue_id}"-def _claim_generation(team_id: int, issue_id: UUID | str, claim: str) -> IssueSummaryOutcome | None:+def _claim_generation(+ team_id: int, issue_id: UUID | str, claim: str, *, fails_closed: bool+) -> IssueSummaryOutcome | None:"""None when this worker now holds the issue under `claim`, else the outcome that holds it."""try:held = get_client().register_script(CLAIM_LUA)(@@ -581,9 +638,9 @@ def _claim_generation(team_id: int, issue_id: UUID | str, claim: str) -> IssueSuargs=[LOCK_QUEUED, claim, ISSUE_SUMMARY_LOCK_TTL_SECONDS],)except RedisError:- # Fail open, as the budget does: a Redis outage can let two concurrent reads both pay.logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)- return None+ # A read fails open, as the budget does: a Redis outage can let two concurrent reads both pay.+ return IssueSummaryOutcome.RATE_LIMITED if fails_closed else Nonereturn _lock_outcome(held) if held is not None else None@@ -595,16 +652,22 @@ def _lock_outcome(held: bytes | str | None) -> IssueSummaryOutcome:return IssueSummaryOutcome(value)-def _settle_claim(team_id: int, issue_id: UUID | str, claim: str, outcome: IssueSummaryOutcome) -> None:+def _settle_claim(+ team_id: int, issue_id: UUID | str, claim: str, outcome: IssueSummaryOutcome, *, release: bool+) -> None:"""Replace this worker's claim with `outcome` when no summary came of it, so reads stop waiting for one.A hold outcome extends the lock. Any other failure keeps the lock TTL, so the retry spacing does not change.+ `release` deletes the claim instead.Nothing is written when the key no longer holds `claim`: the lock expired, or another read took the issue."""if outcome in SETTLED_OUTCOMES:return- ttl = str(ISSUE_SUMMARY_HOLD_SECONDS) if outcome in HOLD_OUTCOMES else ""try:+ if release:+ get_client().register_script(RELEASE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[claim])+ return+ ttl = str(ISSUE_SUMMARY_HOLD_SECONDS) if outcome in HOLD_OUTCOMES else ""get_client().register_script(SETTLE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[claim, outcome.value, ttl])except RedisError:logger.warning("error_tracking_issue_summary_hold_unavailable", team_id=team_id, exc_info=True)diff --git a/products/error_tracking/backend/logic/issue_summary_config.py b/products/error_tracking/backend/logic/issue_summary_config.pyindex b69c01c074b6..4a355c67ecbe 100644--- a/products/error_tracking/backend/logic/issue_summary_config.py+++ b/products/error_tracking/backend/logic/issue_summary_config.py@@ -10,6 +10,8 @@ from posthog.token_bucket import Budgetfrom products.error_tracking.backend.logic.severity_inference import SEVERITY_INFERENCE_MODELISSUE_SUMMARY_FLAG = "error-tracking-issue-summaries"+# The issue-created workflow generates the first summary only when this flag and the read flag are both on.+ISSUE_SUMMARY_ON_WRITE_FLAG = "error-tracking-issue-summaries-on-write"ISSUE_SUMMARY_MODEL = "posthog/zai-org/glm-5.2"ISSUE_SUMMARY_CATEGORY_MODEL = SEVERITY_INFERENCE_MODEL@@ -19,11 +21,16 @@ ISSUE_SUMMARY_LLM_TIMEOUT_SECONDS = 30.0# A team gets 100 summaries at once, then about 120 a day.ISSUE_SUMMARY_BUDGET = Budget(burst=100, per_hour=5)+# Summaries generated when an issue is created spend this budget, so a burst of new issues never drains the read budget.+# The refill sits well above a busy project's hourly rate of new issues, and the burst absorbs a release that creates+# many issues at once.+ISSUE_SUMMARY_WRITE_BUDGET = Budget(burst=200, per_hour=60)# A changed input hash regenerates a summary at most this often.ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL = timedelta(hours=24)-# Holds one generation per issue at a time. It is never released, so it also spaces out retries after a failure.+# Holds one generation per issue at a time. A read never releases its claim, so the lock also spaces out read retries+# after a failure. A write that leaves no summary deletes its claim, so it never delays a read.ISSUE_SUMMARY_LOCK_TTL_SECONDS = 300# Marks a queued task until a worker claims it, so page polls queue at most one task per issue in this window.# A lost task keeps reads on "generating" for this long, so it stays short.diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.pyindex 295c7d020290..487e9f58fbdf 100644--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.py+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.py@@ -3,6 +3,7 @@ from products.error_tracking.backend.temporal.lifecycle.issue_created.activitiesemit_issue_created_internal_event_activity,emit_issue_created_signal_activity,generate_issue_created_embedding_activity,+ generate_issue_created_summary_activity,infer_issue_created_severity_activity,merge_issue_created_fingerprint_activity,persist_issue_created_embedding_activity,@@ -19,6 +20,7 @@ __all__ = ["emit_issue_created_internal_event_activity","emit_issue_created_signal_activity","generate_issue_created_embedding_activity",+ "generate_issue_created_summary_activity","infer_issue_created_severity_activity","merge_issue_created_fingerprint_activity","persist_issue_created_embedding_activity",diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.pyindex 12595c7a958d..623aac413c22 100644--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.py+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.py@@ -1,6 +1,9 @@import json+import threadingfrom datetime import UTC+from typing import TYPE_CHECKING+from django.conf import settingsfrom django.utils.dateparse import parse_datetimeimport httpx@@ -19,6 +22,7 @@ from posthog.models import Teamfrom posthog.temporal.common.posthog_client import is_expected_activity_failurefrom posthog.temporal.common.scoped import scoped_temporalfrom posthog.temporal.common.utils import close_db_connections+from posthog.temporal.common.worker import DEFAULT_MAX_CONCURRENT_TASKSfrom products.error_tracking.backend.logic.severity_inference import (apply_inferred_severity,@@ -54,6 +58,9 @@ from products.ml_inference.backend.facade.contracts import (DecisionsDisabledError,)+if TYPE_CHECKING:+ from products.error_tracking.backend.logic.issue_summaries import IssueSummaryOutcome+logger = structlog.get_logger(__name__)EMBEDDING_MODEL = "text-embedding-3-large-3072"@@ -279,6 +286,48 @@ def infer_issue_created_severity_activity(inputs: IssueCreatedWorkflowInputs) ->return result+# A summary holds an activity slot for the whole model call, and alert dispatch shares this worker's slots.+ISSUE_SUMMARY_WORKER_SLOTS = threading.BoundedSemaphore(+ max(1, (settings.MAX_CONCURRENT_ACTIVITIES or DEFAULT_MAX_CONCURRENT_TASKS) // 4)+)+++def _generate_issue_created_summary(inputs: IssueCreatedWorkflowInputs) -> "IssueSummaryOutcome":+ # issue_summaries imports the lifecycle rendering module, whose package imports this module.+ from products.error_tracking.backend.logic.issue_summaries import ( # noqa: PLC0415+ ISSUE_SUMMARY_OUTCOMES,+ IssueSummaryOutcome,+ IssueSummaryTrigger,+ generate_issue_summary_from_event,+ issue_summaries_on_write_enabled,+ )++ # Checked outside the generator, which reports every attempt as an event, so a team without the flag sends none.+ if not issue_summaries_on_write_enabled(inputs.team_id):+ return IssueSummaryOutcome.DISABLED+ # Skip instead of wait, because a waiting activity still holds its slot. The first read generates the summary.+ if not ISSUE_SUMMARY_WORKER_SLOTS.acquire(blocking=False):+ logger.warning("error_tracking_issue_summary_worker_busy", team_id=inputs.team_id, issue_id=inputs.issue_id)+ ISSUE_SUMMARY_OUTCOMES.labels(+ outcome=IssueSummaryOutcome.RATE_LIMITED.value, trigger=IssueSummaryTrigger.WRITE.value+ ).inc()+ return IssueSummaryOutcome.RATE_LIMITED+ try:+ return generate_issue_summary_from_event(+ inputs.team_id, inputs.issue_id, lambda team: fetch_event_properties(team, inputs)+ ).outcome+ finally:+ ISSUE_SUMMARY_WORKER_SLOTS.release()++[email protected](capture_exceptions=False)+@close_db_connections+def generate_issue_created_summary_activity(inputs: IssueCreatedWorkflowInputs) -> str:+ # Model and output failures return an outcome, so an exception here is an infrastructure failure worth a retry.+ return _generate_issue_created_summary(inputs).value++# The three activities below add no properties of their own, so the shared activity interceptor# is the better reporter: it knows which failures are expected.@activity.defn@@ -368,4 +417,5 @@ ACTIVITIES = [infer_issue_created_severity_activity,emit_issue_created_internal_event_activity,emit_issue_created_signal_activity,+ generate_issue_created_summary_activity,]diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.pyindex f76c4df97332..081f1ff584d8 100644--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py@@ -1,6 +1,9 @@import jsonimport uuid+import asyncioimport dataclasses+from collections import defaultdict+from datetime import timedeltafrom types import SimpleNamespaceimport pytest@@ -10,9 +13,10 @@ from django.test import override_settingsimport requestsfrom temporalio import activity+from temporalio.api.enums.v1 import EventTypefrom temporalio.exceptions import ApplicationErrorfrom temporalio.testing import WorkflowEnvironment-from temporalio.worker import UnsandboxedWorkflowRunner, Worker+from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Workerfrom posthog.helpers.tiktoken_encoding import LLM_TOKEN_COUNT_PROXY_MODEL, get_tiktoken_encoding_for_model@@ -26,6 +30,7 @@ from products.error_tracking.backend.temporal.lifecycle.event_properties importerror_tracking_event_properties_key,fetch_event_properties,)+from products.error_tracking.backend.temporal.lifecycle.issue_created import workflow as issue_created_workflowfrom products.error_tracking.backend.temporal.lifecycle.issue_created.activities import (generate_issue_created_embedding_activity,infer_issue_created_severity_activity,@@ -42,7 +47,10 @@ from products.error_tracking.backend.temporal.lifecycle.issue_created.types impoSeverityInferenceSkipReason,SeveritySource,)-from products.error_tracking.backend.temporal.lifecycle.issue_created.workflow import ErrorTrackingIssueCreatedWorkflow+from products.error_tracking.backend.temporal.lifecycle.issue_created.workflow import (+ SUMMARY_ON_WRITE_PATCH,+ ErrorTrackingIssueCreatedWorkflow,+)from products.error_tracking.backend.temporal.lifecycle.issue_reopened.types import IssueReopenedWorkflowInputsfrom products.error_tracking.backend.temporal.lifecycle.rendering import decode_token_prefix, render_stacktracefrom products.error_tracking.backend.temporal.lifecycle.types import LifecycleIssueSnapshot@@ -399,6 +407,9 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:signal_issue_ids: list[str] = []signal_attempts: dict[str, int] = {}inferred_issue_ids: list[str] = []+ summary_issue_ids: list[str] = []+ alerted: defaultdict[str, asyncio.Event] = defaultdict(asyncio.Event)+ summary_saw_alert: set[str] = set()reopened_alerts: list[IssueReopenedWorkflowInputs] = []reopened_events: list[IssueReopenedWorkflowInputs] = []reopened_signals: list[IssueReopenedWorkflowInputs] = []@@ -462,6 +473,17 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:@activity.defn(name="dispatch_issue_created_alert_activity")async def dispatch_alert(inputs: IssueCreatedWorkflowInputs) -> None:alert_issue_ids.append(inputs.issue_id)+ alerted[inputs.issue_id].set()++ @activity.defn(name="generate_issue_created_summary_activity")+ async def generate_summary(inputs: IssueCreatedWorkflowInputs) -> str:+ summary_issue_ids.append(inputs.issue_id)+ if inputs.fingerprint == "summary-fails":+ raise ApplicationError("summary failed", non_retryable=True)+ if inputs.fingerprint == "summary-waits-for-alert":+ await asyncio.wait_for(alerted[inputs.issue_id].wait(), timeout=10)+ summary_saw_alert.add(inputs.issue_id)+ return "generated"@activity.defn(name="emit_issue_created_internal_event_activity")async def emit_event(inputs: IssueCreatedWorkflowInputs) -> None:@@ -499,6 +521,7 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:merge,infer_severity,dispatch_alert,+ generate_summary,emit_event,emit_signal,dispatch_reopened_alert,@@ -530,6 +553,18 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:id=str(uuid.uuid4()),task_queue=task_queue,)+ summary_fails_inputs = _inputs("summary-fails")+ summary_waits_inputs = _inputs("summary-waits-for-alert")+ summary_results = [+ await environment.client.execute_workflow(+ ErrorTrackingIssueCreatedWorkflow.run,+ summary_inputs,+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ execution_timeout=timedelta(minutes=1),+ )+ for summary_inputs in (summary_fails_inputs, summary_waits_inputs)+ ]reopened_inputs = _inputs("merged-into-resolved")merged_result = await environment.client.execute_workflow(ErrorTrackingIssueCreatedWorkflow.run,@@ -568,7 +603,17 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:inference_unavailable_inputs.issue_id,severity_changed_inputs.issue_id,]- notified_issue_ids = [*severity_issue_ids, unmerged_inputs.issue_id, embedding_unavailable_inputs.issue_id]+ notified_issue_ids = [+ *severity_issue_ids,+ summary_fails_inputs.issue_id,+ summary_waits_inputs.issue_id,+ unmerged_inputs.issue_id,+ embedding_unavailable_inputs.issue_id,+ ]+ # A failed or slow summary neither fails the workflow nor holds the alert, and a merged issue gets no summary.+ assert summary_results == [IssueCreatedWorkflowResult(notified=True)] * 2+ assert summary_saw_alert == {summary_waits_inputs.issue_id}+ assert summary_issue_ids == notified_issue_idsassert alert_issue_ids == notified_issue_idsassert event_issue_ids == notified_issue_idsassert signal_issue_ids == notified_issue_ids@@ -599,3 +644,64 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:assert reopened_alerts == [expected_reopened]assert reopened_events == [expected_reopened]assert reopened_signals == [expected_reopened]+++async def test_replays_a_history_recorded_before_the_summary_activity(monkeypatch: pytest.MonkeyPatch) -> None:+ @activity.defn(name="generate_issue_created_embedding_activity")+ async def generate(_: IssueCreatedWorkflowInputs) -> IssueEmbeddingPreparationResult:+ return IssueEmbeddingPreparationResult(team_exists=True, skipped_reason="manual_fingerprint")++ @activity.defn(name="dispatch_issue_created_alert_activity")+ async def dispatch_alert(_: IssueCreatedWorkflowInputs) -> None:+ return None++ @activity.defn(name="emit_issue_created_internal_event_activity")+ async def emit_event(_: IssueCreatedWorkflowInputs) -> None:+ return None++ @activity.defn(name="emit_issue_created_signal_activity")+ async def emit_signal(_: IssueCreatedWorkflowInputs) -> None:+ return None++ @activity.defn(name="generate_issue_created_summary_activity")+ async def generate_summary(_: IssueCreatedWorkflowInputs) -> str:+ return "generated"++ original_patched = issue_created_workflow.workflow.patched++ def before_summary_patch(patch_id: str) -> bool:+ return False if patch_id == SUMMARY_ON_WRITE_PATCH else original_patched(patch_id)++ task_queue = str(uuid.uuid4())+ async with await WorkflowEnvironment.start_time_skipping() as environment:+ async with Worker(+ environment.client,+ task_queue=task_queue,+ workflows=[ErrorTrackingIssueCreatedWorkflow],+ activities=[generate, dispatch_alert, emit_event, emit_signal, generate_summary],+ workflow_runner=UnsandboxedWorkflowRunner(),+ ):+ with monkeypatch.context() as patched:+ patched.setattr(issue_created_workflow.workflow, "patched", before_summary_patch)+ handle = await environment.client.start_workflow(+ ErrorTrackingIssueCreatedWorkflow.run,+ _inputs("unmerged"),+ id=str(uuid.uuid4()),+ task_queue=task_queue,+ execution_timeout=timedelta(minutes=1),+ )+ assert await handle.result() == IssueCreatedWorkflowResult(+ notified=True, embedding_skipped_reason="manual_fingerprint"+ )+ history = await handle.fetch_history()++ scheduled = [+ event.activity_task_scheduled_event_attributes.activity_type.name+ for event in history.events+ if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED+ ]+ assert "generate_issue_created_summary_activity" not in scheduled+ await Replayer(+ workflows=[ErrorTrackingIssueCreatedWorkflow], workflow_runner=UnsandboxedWorkflowRunner()+ ).replay_workflow(history)diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.pyindex e97566c2f0f8..f77cd6edd278 100644--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.py+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.py@@ -41,6 +41,15 @@ SEVERITY_INFERENCE_RETRY_POLICY = common.RetryPolicy(SEVERITY_INFERENCE_START_TO_CLOSE_TIMEOUT = timedelta(seconds=35)# Alerts wait for inference, so a model outage must not hold them past this bound, retries included.SEVERITY_INFERENCE_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(seconds=45)+SUMMARY_ON_WRITE_PATCH = "error-tracking-summary-on-write-activity"+SUMMARY_RETRY_POLICY = common.RetryPolicy(+ initial_interval=timedelta(seconds=5),+ maximum_interval=timedelta(seconds=30),+ maximum_attempts=3,+)+# Sized past one model call (ISSUE_SUMMARY_LLM_TIMEOUT_SECONDS) plus the category call and the input build.+SUMMARY_START_TO_CLOSE_TIMEOUT = timedelta(seconds=60)+SUMMARY_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(minutes=5)EMBEDDING_ACTIVITY_RETRY_POLICY = common.RetryPolicy(initial_interval=timedelta(seconds=1),maximum_interval=timedelta(seconds=15),@@ -141,6 +150,11 @@ class ErrorTrackingIssueCreatedWorkflow(PostHogWorkflow):if workflow.patched(ALERT_DISPATCH_PATCH)else None)+ # Patched for the same reason. The alert and the emit activities do not wait for the summary.+ # The workflow awaits it before it completes, because a task left unawaited is dropped when the workflow returns.+ summary = (+ asyncio.create_task(self._generate_summary(inputs)) if workflow.patched(SUMMARY_ON_WRITE_PATCH) else None+ )try:await workflow.execute_activity("emit_issue_created_internal_event_activity",@@ -157,6 +171,8 @@ class ErrorTrackingIssueCreatedWorkflow(PostHogWorkflow):finally:if dispatch is not None:await dispatch+ if summary is not None:+ await summaryreturn IssueCreatedWorkflowResult(notified=True,embedding_skipped_reason=preparation.skipped_reason,@@ -190,6 +206,26 @@ class ErrorTrackingIssueCreatedWorkflow(PostHogWorkflow):return inputsreturn dataclasses.replace(inputs, issue=dataclasses.replace(inputs.issue, severity=result.stored_severity))+ async def _generate_summary(self, inputs: IssueCreatedWorkflowInputs) -> None:+ try:+ await workflow.execute_activity(+ "generate_issue_created_summary_activity",+ inputs,+ schedule_to_close_timeout=SUMMARY_SCHEDULE_TO_CLOSE_TIMEOUT,+ start_to_close_timeout=SUMMARY_START_TO_CLOSE_TIMEOUT,+ retry_policy=SUMMARY_RETRY_POLICY,+ )+ except ActivityError:+ workflow.logger.warning("Issue summary generation failed; a read generates it instead")+ (+ workflow.metric_meter()+ .create_counter(+ "error_tracking_issue_created_summary_fail_open",+ "Issue-created workflows whose summary activity failed",+ )+ .add(1)+ )+def _reopened_inputs(inputs: IssueCreatedWorkflowInputs, target: AutoMergeReopenedTargetdiff --git a/products/error_tracking/backend/tests/test_issue_summaries.py b/products/error_tracking/backend/tests/test_issue_summaries.pyindex 1ec5e6f03564..018455b40cff 100644--- a/products/error_tracking/backend/tests/test_issue_summaries.py+++ b/products/error_tracking/backend/tests/test_issue_summaries.py@@ -1,6 +1,7 @@import jsonimport signalimport socket+import threadingfrom contextlib import nullcontextfrom datetime import UTC, datetime, timedeltafrom typing import Any@@ -36,12 +37,21 @@ from products.error_tracking.backend.logic.issue_summaries import (get_or_generate_issue_summary,parse_summary_response,)+from products.error_tracking.backend.logic.issue_summary_config import ISSUE_SUMMARY_FLAG, ISSUE_SUMMARY_ON_WRITE_FLAGfrom products.error_tracking.backend.models import (ErrorTrackingIssue,ErrorTrackingIssueFingerprintV2,ErrorTrackingIssueSummary,sync_issues_to_clickhouse,)+from products.error_tracking.backend.temporal.lifecycle.issue_created import activities as issue_created_activities+from products.error_tracking.backend.temporal.lifecycle.issue_created.activities import (+ generate_issue_created_summary_activity,+)+from products.error_tracking.backend.temporal.lifecycle.issue_created.types import (+ IssueCreatedSnapshot,+ IssueCreatedWorkflowInputs,+)from products.ml_inference.backend.facade.contracts import ChoiceAnswer, DecisionResultSUMMARY_REPLY = {@@ -192,16 +202,30 @@ class TestIssueSummaryGeneration(BaseTest):@parameterized.expand([- ("lock_unavailable", "get_client", {"side_effect": RedisError("down")}),- ("budget_unavailable", "consume", {"return_value": BucketUnavailable(error="down")}),+ (f"{name}_{trigger}", target, patch_kwargs, trigger, expected)+ for name, target, patch_kwargs in [+ ("lock_unavailable", "get_client", {"side_effect": RedisError("down")}),+ ("budget_peek_unavailable", "peek", {"return_value": BucketUnavailable(error="down")}),+ ("budget_unavailable", "consume", {"return_value": BucketUnavailable(error="down")}),+ ]+ for trigger, expected in [+ ("read", IssueSummaryOutcome.GENERATED),+ ("write", IssueSummaryOutcome.RATE_LIMITED),+ ]])- def test_generates_when_redis_is_down(self, _name: str, target: str, patch_kwargs: dict[str, Any]) -> None:+ def test_redis_outage_fails_open_on_read_and_closed_on_write(+ self, _name: str, target: str, patch_kwargs: dict[str, Any], trigger: str, expected: IssueSummaryOutcome+ ) -> None:self.enterContext(patch.object(issue_summaries, target, **patch_kwargs))- result = get_or_generate_issue_summary(self.team.id, self.issue.id)+ if trigger == "read":+ outcome = get_or_generate_issue_summary(self.team.id, self.issue.id).outcome+ else:+ outcome = self._generate_on_creation()- assert result.outcome == IssueSummaryOutcome.GENERATED+ assert outcome == expected+ assert self.llm_calls == (1 if expected == IssueSummaryOutcome.GENERATED else 0)@parameterized.expand([@@ -227,6 +251,7 @@ class TestIssueSummaryGeneration(BaseTest):properties = capture.call_args.kwargs["properties"]assert properties["outcome"] == outcome.value+ assert properties["trigger"] == "read"assert properties["queue_ms"] == 2000assert stages <= {key for key in properties if key.endswith("_ms")}@@ -355,16 +380,8 @@ class TestIssueSummaryGeneration(BaseTest):assert shown is not Noneassert (shown.generated_at, shown.checked_at) == (result.summary.generated_at, result.summary.checked_at)- def _generate_on_creation(- self, budget_key: str | None = None, budget: Budget = issue_summaries.ISSUE_SUMMARY_BUDGET- ) -> IssueSummaryOutcome:- return generate_issue_summary_from_event(- self.team.id,- self.issue.id,- {},- budget_key=budget_key or self._budget_key,- budget=budget,- ).outcome+ def _generate_on_creation(self) -> IssueSummaryOutcome:+ return generate_issue_summary_from_event(self.team.id, self.issue.id, lambda _team: {}).outcomedef test_generation_on_creation_runs_once_and_keeps_a_stored_summary(self) -> None:with time_machine.travel(NOW, tick=False):@@ -387,16 +404,102 @@ class TestIssueSummaryGeneration(BaseTest):assert capture.call_args.kwargs["event"] == issue_summaries.ISSUE_SUMMARY_TASK_EVENTproperties = capture.call_args.kwargs["properties"]assert properties["outcome"] == IssueSummaryOutcome.GENERATED.value+ assert properties["trigger"] == "write"assert {"total_ms", "inputs_ms", "model_ms"} <= properties.keys()- def test_generation_on_creation_spends_the_budget_its_caller_passes(self) -> None:- creation_key = f"{self._budget_key}:on-creation"- tiny = Budget(burst=1, per_hour=1)- consume(creation_key, tiny)+ @parameterized.expand(+ [+ ("own_team_drained", 0, IssueSummaryOutcome.RATE_LIMITED),+ ("other_team_drained", 1, IssueSummaryOutcome.GENERATED),+ ]+ )+ def test_write_budget_is_per_team_and_never_holds_a_read(+ self, _name: str, drained_team_offset: int, expected: IssueSummaryOutcome+ ) -> None:+ self._drain_write_budget(self.team.id + drained_team_offset)++ assert self._generate_on_creation() == expected+ read = get_or_generate_issue_summary(self.team.id, self.issue.id)++ assert read.summary is not None+ assert self.llm_calls == 1++ @parameterized.expand(+ [+ ("on_write_flag_off", {ISSUE_SUMMARY_ON_WRITE_FLAG}, None, IssueSummaryOutcome.DISABLED),+ ("read_flag_off", {ISSUE_SUMMARY_FLAG}, None, IssueSummaryOutcome.DISABLED),+ ("ai_not_approved", set(), lambda test: test._disapprove_ai(), IssueSummaryOutcome.AI_NOT_APPROVED),+ (+ "write_budget_drained",+ set(),+ lambda test: test._drain_write_budget(test.team.id),+ IssueSummaryOutcome.RATE_LIMITED,+ ),+ ("all_checks_pass", set(), None, IssueSummaryOutcome.GENERATED),+ ]+ )+ def test_issue_created_activity_fails_closed(+ self, _name: str, flags_off: set[str], arrange: Any, expected: IssueSummaryOutcome+ ) -> None:+ self.enterContext(+ patch.object(+ team_flags, "feature_enabled_or_false", side_effect=lambda flag, *_, **__: flag not in flags_off+ )+ )+ fetch = self.enterContext(+ patch.object(issue_created_activities, "fetch_event_properties", return_value={"$lib": "web"})+ )+ capture = MagicMock()+ self.enterContext(patch.object(issue_summaries, "ph_background_capture", return_value=capture))+ if arrange is not None:+ arrange(self)++ outcome = generate_issue_created_summary_activity(self._issue_created_inputs())++ assert outcome == expected.value+ assert self.llm_calls == (1 if expected == IssueSummaryOutcome.GENERATED else 0)+ assert fetch.called is (expected == IssueSummaryOutcome.GENERATED)+ assert capture.called is (ISSUE_SUMMARY_ON_WRITE_FLAG not in flags_off)++ def test_issue_created_activity_skips_while_worker_summary_slots_are_full(self) -> None:+ self.enterContext(patch.object(team_flags, "feature_enabled_or_false", return_value=True))+ self.enterContext(+ patch.object(issue_created_activities, "fetch_event_properties", return_value={"$lib": "web"})+ )+ slots = threading.BoundedSemaphore(1)+ self.enterContext(patch.object(issue_created_activities, "ISSUE_SUMMARY_WORKER_SLOTS", slots))- assert self._generate_on_creation(budget_key=creation_key, budget=tiny) == IssueSummaryOutcome.RATE_LIMITED+ slots.acquire()+ assert (+ generate_issue_created_summary_activity(self._issue_created_inputs())+ == IssueSummaryOutcome.RATE_LIMITED.value+ )assert self.llm_calls == 0+ slots.release()+ assert (+ generate_issue_created_summary_activity(self._issue_created_inputs()) == IssueSummaryOutcome.GENERATED.value+ )+ assert slots.acquire(blocking=False)++ def _drain_write_budget(self, team_id: int) -> None:+ tiny = Budget(burst=1, per_hour=1)+ self.enterContext(patch.object(issue_summaries, "ISSUE_SUMMARY_WRITE_BUDGET", tiny))+ consume(f"{issue_summaries.ISSUE_SUMMARY_WRITE_BUDGET_KEY_PREFIX}:{team_id}", tiny)++ def _issue_created_inputs(self) -> IssueCreatedWorkflowInputs:+ return IssueCreatedWorkflowInputs(+ notification_id="notification",+ team_id=self.team.id,+ issue_id=str(self.issue.id),+ issue=IssueCreatedSnapshot(+ name="TypeError", description=None, status="active", created_at=NOW.isoformat(), severity=None+ ),+ fingerprint="fp",+ event_uuid="00000000-0000-0000-0000-000000000001",+ event_timestamp=NOW.isoformat(),+ )+def test_merge_keeps_target_summary_and_deletes_source_summary(self) -> None:source = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")ErrorTrackingIssueFingerprintV2.objects.create(team=self.team, issue=source, fingerprint="fp-source")