Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(error-tracking): generate the first issue summary when the issue is created (#124778)

James Gillc22329b75d42
diff --git a/products/error_tracking/backend/logic/issue_summaries.py b/products/error_tracking/backend/logic/issue_summaries.py
index e9e1298ea893..5345a66a8e1c 100644
--- a/products/error_tracking/backend/logic/issue_summaries.py
+++ b/products/error_tracking/backend/logic/issue_summaries.py
@@ -75,10 +75,12 @@ from products.error_tracking.backend.logic.issue_summary_config import (
ISSUE_SUMMARY_MAX_OUTPUT_TOKENS,
ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL,
ISSUE_SUMMARY_MODEL,
+ ISSUE_SUMMARY_ON_WRITE_FLAG,
ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,
ISSUE_SUMMARY_QUEUED_TTL_SECONDS,
ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS,
ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS,
+ ISSUE_SUMMARY_WRITE_BUDGET,
)
from products.error_tracking.backend.logic.issue_summary_prompt import (
ISSUE_SUMMARY_CATEGORY_QUESTION,
@@ -107,6 +109,7 @@ from products.ml_inference.backend.facade.contracts import ChoiceAnswer, Decisio
logger = structlog.get_logger(__name__)
ISSUE_SUMMARY_BUDGET_KEY_PREFIX = "error_tracking_issue_summary"
+ISSUE_SUMMARY_WRITE_BUDGET_KEY_PREFIX = "error_tracking_issue_summary_write"
ISSUE_SUMMARY_LOCK_KEY_PREFIX = "error_tracking_issue_summary_lock"
ISSUE_SUMMARY_AI_PRODUCT = "error_tracking"
ISSUE_SUMMARY_TASK_EVENT = "error_tracking_issue_summary_generation"
@@ -144,8 +147,8 @@ else
end
return 1
"""
-# Deletes the key only while it holds the queued marker, so a worker's claim or a hold outcome survives.
-RELEASE_QUEUED_LUA = """
+# Deletes the key only while it holds ARGV[1], so a value another caller wrote survives.
+RELEASE_LUA = """
if redis.call('GET', KEYS[1]) == ARGV[1] then
return redis.call('DEL', KEYS[1])
end
@@ -153,6 +156,11 @@ return 0
"""
+class IssueSummaryTrigger(StrEnum):
+ READ = "read"
+ WRITE = "write"
+
+
class IssueSummaryOutcome(StrEnum):
GENERATED = "generated"
# A stored summary is younger than the regenerate interval.
@@ -179,8 +187,8 @@ SETTLED_OUTCOMES = frozenset({IssueSummaryOutcome.GENERATED, IssueSummaryOutcome
ISSUE_SUMMARY_OUTCOMES = Counter(
"error_tracking_issue_summary_total",
- "Issue summary requests by outcome",
- labelnames=("outcome",),
+ "Issue summary requests by outcome and trigger",
+ labelnames=("outcome", "trigger"),
)
ISSUE_SUMMARY_STAGE_SECONDS = Histogram(
@@ -231,6 +239,10 @@ def issue_summaries_enabled(team_id: int) -> bool:
return team_flag_enabled(ISSUE_SUMMARY_FLAG, team_id)
+def issue_summaries_on_write_enabled(team_id: int) -> bool:
+ return team_flag_enabled(ISSUE_SUMMARY_ON_WRITE_FLAG, team_id)
+
+
def summary_sources_readable(team_id: int, user: User) -> bool:
"""False when the user may not read an event property the summary is written from.
@@ -306,7 +318,7 @@ def _queue_generation(team_id: int, issue_id: UUID | str) -> bool:
capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)})
# Without a task, the queued marker would keep reads waiting until it expires.
try:
- _start_client().register_script(RELEASE_QUEUED_LUA)(keys=[_lock_key(team_id, issue_id)], args=[LOCK_QUEUED])
+ _start_client().register_script(RELEASE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[LOCK_QUEUED])
except RedisError:
logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)
return False
@@ -366,42 +378,68 @@ def run_issue_summary_task(team_id: int, issue_id: UUID | str, queued_at: float
timings = IssueSummaryTimings()
if queued_at is not None:
timings.record("queue", max(time.time() - queued_at, 0))
- _generate_and_report(team_id, issue_id, timings, lambda: get_or_generate_issue_summary(team_id, issue_id, timings))
+ _generate_and_report(
+ team_id,
+ issue_id,
+ IssueSummaryTrigger.READ,
+ timings,
+ lambda: get_or_generate_issue_summary(team_id, issue_id, timings),
+ )
def generate_issue_summary_from_event(
- team_id: int, issue_id: UUID | str, event_properties: dict[str, object], *, budget_key: str, budget: Budget
+ team_id: int, issue_id: UUID | str, load_event_properties: Callable[[Team], dict[str, object]]
) -> IssueSummaryResult:
"""Generate the first summary of a new issue from its first event, with no ClickHouse query.
Does nothing when a summary is already stored. A stored summary refreshes through the read path.
+ `load_event_properties` runs only after every check passes, so a blocked attempt reads no event.
"""
generation = _SummaryGeneration(
+ trigger=IssueSummaryTrigger.WRITE,
due=first_summary_due,
- budget_key=budget_key,
- budget=budget,
- build_inputs=lambda team, _timings: build_issue_summary_inputs_from_event(team, issue_id, event_properties),
+ budget_key=f"{ISSUE_SUMMARY_WRITE_BUDGET_KEY_PREFIX}:{team_id}",
+ budget=ISSUE_SUMMARY_WRITE_BUDGET,
+ build_inputs=lambda team, _timings: build_issue_summary_inputs_from_event(
+ team, issue_id, load_event_properties(team)
+ ),
)
timings = IssueSummaryTimings()
return _generate_and_report(
- team_id, issue_id, timings, lambda: _get_or_generate_counted(team_id, issue_id, generation, timings)
+ team_id,
+ issue_id,
+ generation.trigger,
+ timings,
+ lambda: _get_or_generate_counted(team_id, issue_id, generation, timings),
)
def _generate_and_report(
- team_id: int, issue_id: UUID | str, timings: IssueSummaryTimings, generate: Callable[[], IssueSummaryResult]
+ team_id: int,
+ issue_id: UUID | str,
+ trigger: IssueSummaryTrigger,
+ timings: IssueSummaryTimings,
+ generate: Callable[[], IssueSummaryResult],
) -> IssueSummaryResult:
try:
result = generate()
except Exception:
- _report_task(team_id, issue_id, IssueSummaryOutcome.UNEXPECTED_ERROR, has_summary=False, timings=timings)
+ _report_task(
+ team_id, issue_id, trigger, IssueSummaryOutcome.UNEXPECTED_ERROR, has_summary=False, timings=timings
+ )
raise
- _report_task(team_id, issue_id, result.outcome, has_summary=result.summary is not None, timings=timings)
+ _report_task(team_id, issue_id, trigger, result.outcome, has_summary=result.summary is not None, timings=timings)
return result
def _report_task(
- team_id: int, issue_id: UUID | str, outcome: IssueSummaryOutcome, *, has_summary: bool, timings: IssueSummaryTimings
+ team_id: int,
+ issue_id: UUID | str,
+ trigger: IssueSummaryTrigger,
+ outcome: IssueSummaryOutcome,
+ *,
+ has_summary: bool,
+ timings: IssueSummaryTimings,
) -> None:
# The background client sends from its own thread, so the worker takes the next task without waiting for a flush.
try:
@@ -413,6 +451,7 @@ def _report_task(
"issue_id": str(issue_id),
# The client's `region` super property names the destination project and overwrites an event `region`.
"instance_region": get_instance_region(),
+ "trigger": trigger.value,
"outcome": outcome.value,
"has_summary": has_summary,
**timings.ms,
@@ -430,6 +469,7 @@ def get_or_generate_issue_summary(
Generation runs at most once at a time per issue. A failure keeps the stored summary.
"""
generation = _SummaryGeneration(
+ trigger=IssueSummaryTrigger.READ,
due=regeneration_due,
budget_key=f"{ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{team_id}",
budget=ISSUE_SUMMARY_BUDGET,
@@ -442,11 +482,16 @@ def get_or_generate_issue_summary(
class _SummaryGeneration:
"""What differs between a generation on read and one on issue creation."""
+ trigger: IssueSummaryTrigger
due: DueCheck
budget_key: str
budget: Budget
build_inputs: Callable[[Team, IssueSummaryTimings], IssueSummaryInputs | IssueSummaryOutcome]
+ @property
+ def fails_closed(self) -> bool:
+ return self.trigger == IssueSummaryTrigger.WRITE
+
def _get_or_generate_counted(
team_id: int, issue_id: UUID | str, generation: _SummaryGeneration, timings: IssueSummaryTimings
@@ -455,14 +500,17 @@ def _get_or_generate_counted(
with timings.stage("total"):
result = _get_or_generate(team_id, issue_id, generation, timings)
except Exception:
- ISSUE_SUMMARY_OUTCOMES.labels(outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value).inc()
+ ISSUE_SUMMARY_OUTCOMES.labels(
+ outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value, trigger=generation.trigger.value
+ ).inc()
raise
- ISSUE_SUMMARY_OUTCOMES.labels(outcome=result.outcome.value).inc()
+ ISSUE_SUMMARY_OUTCOMES.labels(outcome=result.outcome.value, trigger=generation.trigger.value).inc()
logger.info(
"error_tracking_issue_summary",
team_id=team_id,
issue_id=str(issue_id),
outcome=result.outcome.value,
+ trigger=generation.trigger.value,
has_summary=result.summary is not None,
**timings.ms,
)
@@ -476,15 +524,18 @@ def _get_or_generate(
if (blocked := generation_blocked(team_id, stored, generation.due)) is not None:
return IssueSummaryResult(outcome=blocked, summary=stored)
claim = f"{LOCK_CLAIM_PREFIX}{uuid4().hex}"
- if (held := _claim_generation(team_id, issue_id, claim)) is not None:
+ if (held := _claim_generation(team_id, issue_id, claim, fails_closed=generation.fails_closed)) is not None:
return IssueSummaryResult(outcome=held, summary=stored)
+ # A write attempt that leaves no summary deletes its claim instead of writing its outcome to the lock.
+ # The next read then generates at once, so the write budget or a write failure never blocks a read.
+ release = generation.trigger == IssueSummaryTrigger.WRITE
try:
result = _generate_claimed(team_id, issue_id, generation, timings)
except Exception:
- _settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR)
+ _settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR, release=release)
raise
- _settle_claim(team_id, issue_id, claim, result.outcome)
+ _settle_claim(team_id, issue_id, claim, result.outcome, release=release)
return result
@@ -499,7 +550,7 @@ def _generate_claimed(
budget_key, budget = generation.budget_key, generation.budget
# The input queries run before `consume`, so an unchanged issue spends no token. This read keeps a team
# with no tokens left from running them for every issue it opens. `consume` still makes the decision.
- if _budget_exhausted(budget_key, budget):
+ if _budget_exhausted(budget_key, budget, fails_closed=generation.fails_closed):
return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)
team = Team.objects.get(id=team_id)
@@ -522,8 +573,10 @@ def _generate_claimed(
case BucketDecision(allowed=False):
return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)
case BucketUnavailable(error=bucket_error):
- # Fail open: the per-issue lock and the regenerate interval still bound the spend.
logger.warning("error_tracking_issue_summary_budget_unavailable", team_id=team_id, error=bucket_error)
+ if generation.fails_closed:
+ return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)
+ # A read fails open: the per-issue lock and the regenerate interval still bound the spend.
# Only the Go gateway honours privacy mode, so the Python gateway fallback is never used.
if resolve_ai_gateway_config() is None:
@@ -564,16 +617,20 @@ def _generate_claimed(
return IssueSummaryResult(outcome=IssueSummaryOutcome.GENERATED, summary=summary)
-def _budget_exhausted(budget_key: str, budget: Budget) -> bool:
+def _budget_exhausted(budget_key: str, budget: Budget, *, fails_closed: bool) -> bool:
decision = peek(budget_key, budget)
- return isinstance(decision, BucketDecision) and not decision.allowed
+ if isinstance(decision, BucketUnavailable):
+ return fails_closed
+ return not decision.allowed
def _lock_key(team_id: int, issue_id: UUID | str) -> str:
return f"{ISSUE_SUMMARY_LOCK_KEY_PREFIX}:{team_id}:{issue_id}"
-def _claim_generation(team_id: int, issue_id: UUID | str, claim: str) -> IssueSummaryOutcome | None:
+def _claim_generation(
+ team_id: int, issue_id: UUID | str, claim: str, *, fails_closed: bool
+) -> IssueSummaryOutcome | None:
"""None when this worker now holds the issue under `claim`, else the outcome that holds it."""
try:
held = get_client().register_script(CLAIM_LUA)(
@@ -581,9 +638,9 @@ def _claim_generation(team_id: int, issue_id: UUID | str, claim: str) -> IssueSu
args=[LOCK_QUEUED, claim, ISSUE_SUMMARY_LOCK_TTL_SECONDS],
)
except RedisError:
- # Fail open, as the budget does: a Redis outage can let two concurrent reads both pay.
logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)
- return None
+ # A read fails open, as the budget does: a Redis outage can let two concurrent reads both pay.
+ return IssueSummaryOutcome.RATE_LIMITED if fails_closed else None
return _lock_outcome(held) if held is not None else None
@@ -595,16 +652,22 @@ def _lock_outcome(held: bytes | str | None) -> IssueSummaryOutcome:
return IssueSummaryOutcome(value)
-def _settle_claim(team_id: int, issue_id: UUID | str, claim: str, outcome: IssueSummaryOutcome) -> None:
+def _settle_claim(
+ team_id: int, issue_id: UUID | str, claim: str, outcome: IssueSummaryOutcome, *, release: bool
+) -> None:
"""Replace this worker's claim with `outcome` when no summary came of it, so reads stop waiting for one.
A hold outcome extends the lock. Any other failure keeps the lock TTL, so the retry spacing does not change.
+ `release` deletes the claim instead.
Nothing is written when the key no longer holds `claim`: the lock expired, or another read took the issue.
"""
if outcome in SETTLED_OUTCOMES:
return
- ttl = str(ISSUE_SUMMARY_HOLD_SECONDS) if outcome in HOLD_OUTCOMES else ""
try:
+ if release:
+ get_client().register_script(RELEASE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[claim])
+ return
+ ttl = str(ISSUE_SUMMARY_HOLD_SECONDS) if outcome in HOLD_OUTCOMES else ""
get_client().register_script(SETTLE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[claim, outcome.value, ttl])
except RedisError:
logger.warning("error_tracking_issue_summary_hold_unavailable", team_id=team_id, exc_info=True)
diff --git a/products/error_tracking/backend/logic/issue_summary_config.py b/products/error_tracking/backend/logic/issue_summary_config.py
index b69c01c074b6..4a355c67ecbe 100644
--- a/products/error_tracking/backend/logic/issue_summary_config.py
+++ b/products/error_tracking/backend/logic/issue_summary_config.py
@@ -10,6 +10,8 @@ from posthog.token_bucket import Budget
from products.error_tracking.backend.logic.severity_inference import SEVERITY_INFERENCE_MODEL
ISSUE_SUMMARY_FLAG = "error-tracking-issue-summaries"
+# The issue-created workflow generates the first summary only when this flag and the read flag are both on.
+ISSUE_SUMMARY_ON_WRITE_FLAG = "error-tracking-issue-summaries-on-write"
ISSUE_SUMMARY_MODEL = "posthog/zai-org/glm-5.2"
ISSUE_SUMMARY_CATEGORY_MODEL = SEVERITY_INFERENCE_MODEL
@@ -19,11 +21,16 @@ ISSUE_SUMMARY_LLM_TIMEOUT_SECONDS = 30.0
# A team gets 100 summaries at once, then about 120 a day.
ISSUE_SUMMARY_BUDGET = Budget(burst=100, per_hour=5)
+# Summaries generated when an issue is created spend this budget, so a burst of new issues never drains the read budget.
+# The refill sits well above a busy project's hourly rate of new issues, and the burst absorbs a release that creates
+# many issues at once.
+ISSUE_SUMMARY_WRITE_BUDGET = Budget(burst=200, per_hour=60)
# A changed input hash regenerates a summary at most this often.
ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL = timedelta(hours=24)
-# Holds one generation per issue at a time. It is never released, so it also spaces out retries after a failure.
+# Holds one generation per issue at a time. A read never releases its claim, so the lock also spaces out read retries
+# after a failure. A write that leaves no summary deletes its claim, so it never delays a read.
ISSUE_SUMMARY_LOCK_TTL_SECONDS = 300
# Marks a queued task until a worker claims it, so page polls queue at most one task per issue in this window.
# A lost task keeps reads on "generating" for this long, so it stays short.
diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.py
index 295c7d020290..487e9f58fbdf 100644
--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.py
+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/__init__.py
@@ -3,6 +3,7 @@ from products.error_tracking.backend.temporal.lifecycle.issue_created.activities
emit_issue_created_internal_event_activity,
emit_issue_created_signal_activity,
generate_issue_created_embedding_activity,
+ generate_issue_created_summary_activity,
infer_issue_created_severity_activity,
merge_issue_created_fingerprint_activity,
persist_issue_created_embedding_activity,
@@ -19,6 +20,7 @@ __all__ = [
"emit_issue_created_internal_event_activity",
"emit_issue_created_signal_activity",
"generate_issue_created_embedding_activity",
+ "generate_issue_created_summary_activity",
"infer_issue_created_severity_activity",
"merge_issue_created_fingerprint_activity",
"persist_issue_created_embedding_activity",
diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.py
index 12595c7a958d..623aac413c22 100644
--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.py
+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/activities.py
@@ -1,6 +1,9 @@
import json
+import threading
from datetime import UTC
+from typing import TYPE_CHECKING
+from django.conf import settings
from django.utils.dateparse import parse_datetime
import httpx
@@ -19,6 +22,7 @@ from posthog.models import Team
from posthog.temporal.common.posthog_client import is_expected_activity_failure
from posthog.temporal.common.scoped import scoped_temporal
from posthog.temporal.common.utils import close_db_connections
+from posthog.temporal.common.worker import DEFAULT_MAX_CONCURRENT_TASKS
from products.error_tracking.backend.logic.severity_inference import (
apply_inferred_severity,
@@ -54,6 +58,9 @@ from products.ml_inference.backend.facade.contracts import (
DecisionsDisabledError,
)
+if TYPE_CHECKING:
+ from products.error_tracking.backend.logic.issue_summaries import IssueSummaryOutcome
+
logger = structlog.get_logger(__name__)
EMBEDDING_MODEL = "text-embedding-3-large-3072"
@@ -279,6 +286,48 @@ def infer_issue_created_severity_activity(inputs: IssueCreatedWorkflowInputs) ->
return result
+# A summary holds an activity slot for the whole model call, and alert dispatch shares this worker's slots.
+ISSUE_SUMMARY_WORKER_SLOTS = threading.BoundedSemaphore(
+ max(1, (settings.MAX_CONCURRENT_ACTIVITIES or DEFAULT_MAX_CONCURRENT_TASKS) // 4)
+)
+
+
+def _generate_issue_created_summary(inputs: IssueCreatedWorkflowInputs) -> "IssueSummaryOutcome":
+ # issue_summaries imports the lifecycle rendering module, whose package imports this module.
+ from products.error_tracking.backend.logic.issue_summaries import ( # noqa: PLC0415
+ ISSUE_SUMMARY_OUTCOMES,
+ IssueSummaryOutcome,
+ IssueSummaryTrigger,
+ generate_issue_summary_from_event,
+ issue_summaries_on_write_enabled,
+ )
+
+ # Checked outside the generator, which reports every attempt as an event, so a team without the flag sends none.
+ if not issue_summaries_on_write_enabled(inputs.team_id):
+ return IssueSummaryOutcome.DISABLED
+ # Skip instead of wait, because a waiting activity still holds its slot. The first read generates the summary.
+ if not ISSUE_SUMMARY_WORKER_SLOTS.acquire(blocking=False):
+ logger.warning("error_tracking_issue_summary_worker_busy", team_id=inputs.team_id, issue_id=inputs.issue_id)
+ ISSUE_SUMMARY_OUTCOMES.labels(
+ outcome=IssueSummaryOutcome.RATE_LIMITED.value, trigger=IssueSummaryTrigger.WRITE.value
+ ).inc()
+ return IssueSummaryOutcome.RATE_LIMITED
+ try:
+ return generate_issue_summary_from_event(
+ inputs.team_id, inputs.issue_id, lambda team: fetch_event_properties(team, inputs)
+ ).outcome
+ finally:
+ ISSUE_SUMMARY_WORKER_SLOTS.release()
+
+
[email protected](capture_exceptions=False)
+@close_db_connections
+def generate_issue_created_summary_activity(inputs: IssueCreatedWorkflowInputs) -> str:
+ # Model and output failures return an outcome, so an exception here is an infrastructure failure worth a retry.
+ return _generate_issue_created_summary(inputs).value
+
+
# The three activities below add no properties of their own, so the shared activity interceptor
# is the better reporter: it knows which failures are expected.
@activity.defn
@@ -368,4 +417,5 @@ ACTIVITIES = [
infer_issue_created_severity_activity,
emit_issue_created_internal_event_activity,
emit_issue_created_signal_activity,
+ generate_issue_created_summary_activity,
]
diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py
index f76c4df97332..081f1ff584d8 100644
--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py
+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/test/test_workflow.py
@@ -1,6 +1,9 @@
import json
import uuid
+import asyncio
import dataclasses
+from collections import defaultdict
+from datetime import timedelta
from types import SimpleNamespace
import pytest
@@ -10,9 +13,10 @@ from django.test import override_settings
import requests
from temporalio import activity
+from temporalio.api.enums.v1 import EventType
from temporalio.exceptions import ApplicationError
from temporalio.testing import WorkflowEnvironment
-from temporalio.worker import UnsandboxedWorkflowRunner, Worker
+from temporalio.worker import Replayer, UnsandboxedWorkflowRunner, Worker
from posthog.helpers.tiktoken_encoding import LLM_TOKEN_COUNT_PROXY_MODEL, get_tiktoken_encoding_for_model
@@ -26,6 +30,7 @@ from products.error_tracking.backend.temporal.lifecycle.event_properties import
error_tracking_event_properties_key,
fetch_event_properties,
)
+from products.error_tracking.backend.temporal.lifecycle.issue_created import workflow as issue_created_workflow
from products.error_tracking.backend.temporal.lifecycle.issue_created.activities import (
generate_issue_created_embedding_activity,
infer_issue_created_severity_activity,
@@ -42,7 +47,10 @@ from products.error_tracking.backend.temporal.lifecycle.issue_created.types impo
SeverityInferenceSkipReason,
SeveritySource,
)
-from products.error_tracking.backend.temporal.lifecycle.issue_created.workflow import ErrorTrackingIssueCreatedWorkflow
+from products.error_tracking.backend.temporal.lifecycle.issue_created.workflow import (
+ SUMMARY_ON_WRITE_PATCH,
+ ErrorTrackingIssueCreatedWorkflow,
+)
from products.error_tracking.backend.temporal.lifecycle.issue_reopened.types import IssueReopenedWorkflowInputs
from products.error_tracking.backend.temporal.lifecycle.rendering import decode_token_prefix, render_stacktrace
from products.error_tracking.backend.temporal.lifecycle.types import LifecycleIssueSnapshot
@@ -399,6 +407,9 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:
signal_issue_ids: list[str] = []
signal_attempts: dict[str, int] = {}
inferred_issue_ids: list[str] = []
+ summary_issue_ids: list[str] = []
+ alerted: defaultdict[str, asyncio.Event] = defaultdict(asyncio.Event)
+ summary_saw_alert: set[str] = set()
reopened_alerts: list[IssueReopenedWorkflowInputs] = []
reopened_events: list[IssueReopenedWorkflowInputs] = []
reopened_signals: list[IssueReopenedWorkflowInputs] = []
@@ -462,6 +473,17 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:
@activity.defn(name="dispatch_issue_created_alert_activity")
async def dispatch_alert(inputs: IssueCreatedWorkflowInputs) -> None:
alert_issue_ids.append(inputs.issue_id)
+ alerted[inputs.issue_id].set()
+
+ @activity.defn(name="generate_issue_created_summary_activity")
+ async def generate_summary(inputs: IssueCreatedWorkflowInputs) -> str:
+ summary_issue_ids.append(inputs.issue_id)
+ if inputs.fingerprint == "summary-fails":
+ raise ApplicationError("summary failed", non_retryable=True)
+ if inputs.fingerprint == "summary-waits-for-alert":
+ await asyncio.wait_for(alerted[inputs.issue_id].wait(), timeout=10)
+ summary_saw_alert.add(inputs.issue_id)
+ return "generated"
@activity.defn(name="emit_issue_created_internal_event_activity")
async def emit_event(inputs: IssueCreatedWorkflowInputs) -> None:
@@ -499,6 +521,7 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:
merge,
infer_severity,
dispatch_alert,
+ generate_summary,
emit_event,
emit_signal,
dispatch_reopened_alert,
@@ -530,6 +553,18 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:
id=str(uuid.uuid4()),
task_queue=task_queue,
)
+ summary_fails_inputs = _inputs("summary-fails")
+ summary_waits_inputs = _inputs("summary-waits-for-alert")
+ summary_results = [
+ await environment.client.execute_workflow(
+ ErrorTrackingIssueCreatedWorkflow.run,
+ summary_inputs,
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(minutes=1),
+ )
+ for summary_inputs in (summary_fails_inputs, summary_waits_inputs)
+ ]
reopened_inputs = _inputs("merged-into-resolved")
merged_result = await environment.client.execute_workflow(
ErrorTrackingIssueCreatedWorkflow.run,
@@ -568,7 +603,17 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:
inference_unavailable_inputs.issue_id,
severity_changed_inputs.issue_id,
]
- notified_issue_ids = [*severity_issue_ids, unmerged_inputs.issue_id, embedding_unavailable_inputs.issue_id]
+ notified_issue_ids = [
+ *severity_issue_ids,
+ summary_fails_inputs.issue_id,
+ summary_waits_inputs.issue_id,
+ unmerged_inputs.issue_id,
+ embedding_unavailable_inputs.issue_id,
+ ]
+ # A failed or slow summary neither fails the workflow nor holds the alert, and a merged issue gets no summary.
+ assert summary_results == [IssueCreatedWorkflowResult(notified=True)] * 2
+ assert summary_saw_alert == {summary_waits_inputs.issue_id}
+ assert summary_issue_ids == notified_issue_ids
assert alert_issue_ids == notified_issue_ids
assert event_issue_ids == notified_issue_ids
assert signal_issue_ids == notified_issue_ids
@@ -599,3 +644,64 @@ async def test_only_notifies_for_an_issue_that_was_not_merged() -> None:
assert reopened_alerts == [expected_reopened]
assert reopened_events == [expected_reopened]
assert reopened_signals == [expected_reopened]
+
+
+async def test_replays_a_history_recorded_before_the_summary_activity(monkeypatch: pytest.MonkeyPatch) -> None:
+ @activity.defn(name="generate_issue_created_embedding_activity")
+ async def generate(_: IssueCreatedWorkflowInputs) -> IssueEmbeddingPreparationResult:
+ return IssueEmbeddingPreparationResult(team_exists=True, skipped_reason="manual_fingerprint")
+
+ @activity.defn(name="dispatch_issue_created_alert_activity")
+ async def dispatch_alert(_: IssueCreatedWorkflowInputs) -> None:
+ return None
+
+ @activity.defn(name="emit_issue_created_internal_event_activity")
+ async def emit_event(_: IssueCreatedWorkflowInputs) -> None:
+ return None
+
+ @activity.defn(name="emit_issue_created_signal_activity")
+ async def emit_signal(_: IssueCreatedWorkflowInputs) -> None:
+ return None
+
+ @activity.defn(name="generate_issue_created_summary_activity")
+ async def generate_summary(_: IssueCreatedWorkflowInputs) -> str:
+ return "generated"
+
+ original_patched = issue_created_workflow.workflow.patched
+
+ def before_summary_patch(patch_id: str) -> bool:
+ return False if patch_id == SUMMARY_ON_WRITE_PATCH else original_patched(patch_id)
+
+ task_queue = str(uuid.uuid4())
+ async with await WorkflowEnvironment.start_time_skipping() as environment:
+ async with Worker(
+ environment.client,
+ task_queue=task_queue,
+ workflows=[ErrorTrackingIssueCreatedWorkflow],
+ activities=[generate, dispatch_alert, emit_event, emit_signal, generate_summary],
+ workflow_runner=UnsandboxedWorkflowRunner(),
+ ):
+ with monkeypatch.context() as patched:
+ patched.setattr(issue_created_workflow.workflow, "patched", before_summary_patch)
+ handle = await environment.client.start_workflow(
+ ErrorTrackingIssueCreatedWorkflow.run,
+ _inputs("unmerged"),
+ id=str(uuid.uuid4()),
+ task_queue=task_queue,
+ execution_timeout=timedelta(minutes=1),
+ )
+ assert await handle.result() == IssueCreatedWorkflowResult(
+ notified=True, embedding_skipped_reason="manual_fingerprint"
+ )
+ history = await handle.fetch_history()
+
+ scheduled = [
+ event.activity_task_scheduled_event_attributes.activity_type.name
+ for event in history.events
+ if event.event_type == EventType.EVENT_TYPE_ACTIVITY_TASK_SCHEDULED
+ ]
+ assert "generate_issue_created_summary_activity" not in scheduled
+ await Replayer(
+ workflows=[ErrorTrackingIssueCreatedWorkflow], workflow_runner=UnsandboxedWorkflowRunner()
+ ).replay_workflow(history)
diff --git a/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.py b/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.py
index e97566c2f0f8..f77cd6edd278 100644
--- a/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.py
+++ b/products/error_tracking/backend/temporal/lifecycle/issue_created/workflow.py
@@ -41,6 +41,15 @@ SEVERITY_INFERENCE_RETRY_POLICY = common.RetryPolicy(
SEVERITY_INFERENCE_START_TO_CLOSE_TIMEOUT = timedelta(seconds=35)
# Alerts wait for inference, so a model outage must not hold them past this bound, retries included.
SEVERITY_INFERENCE_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(seconds=45)
+SUMMARY_ON_WRITE_PATCH = "error-tracking-summary-on-write-activity"
+SUMMARY_RETRY_POLICY = common.RetryPolicy(
+ initial_interval=timedelta(seconds=5),
+ maximum_interval=timedelta(seconds=30),
+ maximum_attempts=3,
+)
+# Sized past one model call (ISSUE_SUMMARY_LLM_TIMEOUT_SECONDS) plus the category call and the input build.
+SUMMARY_START_TO_CLOSE_TIMEOUT = timedelta(seconds=60)
+SUMMARY_SCHEDULE_TO_CLOSE_TIMEOUT = timedelta(minutes=5)
EMBEDDING_ACTIVITY_RETRY_POLICY = common.RetryPolicy(
initial_interval=timedelta(seconds=1),
maximum_interval=timedelta(seconds=15),
@@ -141,6 +150,11 @@ class ErrorTrackingIssueCreatedWorkflow(PostHogWorkflow):
if workflow.patched(ALERT_DISPATCH_PATCH)
else None
)
+ # Patched for the same reason. The alert and the emit activities do not wait for the summary.
+ # The workflow awaits it before it completes, because a task left unawaited is dropped when the workflow returns.
+ summary = (
+ asyncio.create_task(self._generate_summary(inputs)) if workflow.patched(SUMMARY_ON_WRITE_PATCH) else None
+ )
try:
await workflow.execute_activity(
"emit_issue_created_internal_event_activity",
@@ -157,6 +171,8 @@ class ErrorTrackingIssueCreatedWorkflow(PostHogWorkflow):
finally:
if dispatch is not None:
await dispatch
+ if summary is not None:
+ await summary
return IssueCreatedWorkflowResult(
notified=True,
embedding_skipped_reason=preparation.skipped_reason,
@@ -190,6 +206,26 @@ class ErrorTrackingIssueCreatedWorkflow(PostHogWorkflow):
return inputs
return dataclasses.replace(inputs, issue=dataclasses.replace(inputs.issue, severity=result.stored_severity))
+ async def _generate_summary(self, inputs: IssueCreatedWorkflowInputs) -> None:
+ try:
+ await workflow.execute_activity(
+ "generate_issue_created_summary_activity",
+ inputs,
+ schedule_to_close_timeout=SUMMARY_SCHEDULE_TO_CLOSE_TIMEOUT,
+ start_to_close_timeout=SUMMARY_START_TO_CLOSE_TIMEOUT,
+ retry_policy=SUMMARY_RETRY_POLICY,
+ )
+ except ActivityError:
+ workflow.logger.warning("Issue summary generation failed; a read generates it instead")
+ (
+ workflow.metric_meter()
+ .create_counter(
+ "error_tracking_issue_created_summary_fail_open",
+ "Issue-created workflows whose summary activity failed",
+ )
+ .add(1)
+ )
+
def _reopened_inputs(
inputs: IssueCreatedWorkflowInputs, target: AutoMergeReopenedTarget
diff --git a/products/error_tracking/backend/tests/test_issue_summaries.py b/products/error_tracking/backend/tests/test_issue_summaries.py
index 1ec5e6f03564..018455b40cff 100644
--- a/products/error_tracking/backend/tests/test_issue_summaries.py
+++ b/products/error_tracking/backend/tests/test_issue_summaries.py
@@ -1,6 +1,7 @@
import json
import signal
import socket
+import threading
from contextlib import nullcontext
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -36,12 +37,21 @@ from products.error_tracking.backend.logic.issue_summaries import (
get_or_generate_issue_summary,
parse_summary_response,
)
+from products.error_tracking.backend.logic.issue_summary_config import ISSUE_SUMMARY_FLAG, ISSUE_SUMMARY_ON_WRITE_FLAG
from products.error_tracking.backend.models import (
ErrorTrackingIssue,
ErrorTrackingIssueFingerprintV2,
ErrorTrackingIssueSummary,
sync_issues_to_clickhouse,
)
+from products.error_tracking.backend.temporal.lifecycle.issue_created import activities as issue_created_activities
+from products.error_tracking.backend.temporal.lifecycle.issue_created.activities import (
+ generate_issue_created_summary_activity,
+)
+from products.error_tracking.backend.temporal.lifecycle.issue_created.types import (
+ IssueCreatedSnapshot,
+ IssueCreatedWorkflowInputs,
+)
from products.ml_inference.backend.facade.contracts import ChoiceAnswer, DecisionResult
SUMMARY_REPLY = {
@@ -192,16 +202,30 @@ class TestIssueSummaryGeneration(BaseTest):
@parameterized.expand(
[
- ("lock_unavailable", "get_client", {"side_effect": RedisError("down")}),
- ("budget_unavailable", "consume", {"return_value": BucketUnavailable(error="down")}),
+ (f"{name}_{trigger}", target, patch_kwargs, trigger, expected)
+ for name, target, patch_kwargs in [
+ ("lock_unavailable", "get_client", {"side_effect": RedisError("down")}),
+ ("budget_peek_unavailable", "peek", {"return_value": BucketUnavailable(error="down")}),
+ ("budget_unavailable", "consume", {"return_value": BucketUnavailable(error="down")}),
+ ]
+ for trigger, expected in [
+ ("read", IssueSummaryOutcome.GENERATED),
+ ("write", IssueSummaryOutcome.RATE_LIMITED),
+ ]
]
)
- def test_generates_when_redis_is_down(self, _name: str, target: str, patch_kwargs: dict[str, Any]) -> None:
+ def test_redis_outage_fails_open_on_read_and_closed_on_write(
+ self, _name: str, target: str, patch_kwargs: dict[str, Any], trigger: str, expected: IssueSummaryOutcome
+ ) -> None:
self.enterContext(patch.object(issue_summaries, target, **patch_kwargs))
- result = get_or_generate_issue_summary(self.team.id, self.issue.id)
+ if trigger == "read":
+ outcome = get_or_generate_issue_summary(self.team.id, self.issue.id).outcome
+ else:
+ outcome = self._generate_on_creation()
- assert result.outcome == IssueSummaryOutcome.GENERATED
+ assert outcome == expected
+ assert self.llm_calls == (1 if expected == IssueSummaryOutcome.GENERATED else 0)
@parameterized.expand(
[
@@ -227,6 +251,7 @@ class TestIssueSummaryGeneration(BaseTest):
properties = capture.call_args.kwargs["properties"]
assert properties["outcome"] == outcome.value
+ assert properties["trigger"] == "read"
assert properties["queue_ms"] == 2000
assert stages <= {key for key in properties if key.endswith("_ms")}
@@ -355,16 +380,8 @@ class TestIssueSummaryGeneration(BaseTest):
assert shown is not None
assert (shown.generated_at, shown.checked_at) == (result.summary.generated_at, result.summary.checked_at)
- def _generate_on_creation(
- self, budget_key: str | None = None, budget: Budget = issue_summaries.ISSUE_SUMMARY_BUDGET
- ) -> IssueSummaryOutcome:
- return generate_issue_summary_from_event(
- self.team.id,
- self.issue.id,
- {},
- budget_key=budget_key or self._budget_key,
- budget=budget,
- ).outcome
+ def _generate_on_creation(self) -> IssueSummaryOutcome:
+ return generate_issue_summary_from_event(self.team.id, self.issue.id, lambda _team: {}).outcome
def test_generation_on_creation_runs_once_and_keeps_a_stored_summary(self) -> None:
with time_machine.travel(NOW, tick=False):
@@ -387,16 +404,102 @@ class TestIssueSummaryGeneration(BaseTest):
assert capture.call_args.kwargs["event"] == issue_summaries.ISSUE_SUMMARY_TASK_EVENT
properties = capture.call_args.kwargs["properties"]
assert properties["outcome"] == IssueSummaryOutcome.GENERATED.value
+ assert properties["trigger"] == "write"
assert {"total_ms", "inputs_ms", "model_ms"} <= properties.keys()
- def test_generation_on_creation_spends_the_budget_its_caller_passes(self) -> None:
- creation_key = f"{self._budget_key}:on-creation"
- tiny = Budget(burst=1, per_hour=1)
- consume(creation_key, tiny)
+ @parameterized.expand(
+ [
+ ("own_team_drained", 0, IssueSummaryOutcome.RATE_LIMITED),
+ ("other_team_drained", 1, IssueSummaryOutcome.GENERATED),
+ ]
+ )
+ def test_write_budget_is_per_team_and_never_holds_a_read(
+ self, _name: str, drained_team_offset: int, expected: IssueSummaryOutcome
+ ) -> None:
+ self._drain_write_budget(self.team.id + drained_team_offset)
+
+ assert self._generate_on_creation() == expected
+ read = get_or_generate_issue_summary(self.team.id, self.issue.id)
+
+ assert read.summary is not None
+ assert self.llm_calls == 1
+
+ @parameterized.expand(
+ [
+ ("on_write_flag_off", {ISSUE_SUMMARY_ON_WRITE_FLAG}, None, IssueSummaryOutcome.DISABLED),
+ ("read_flag_off", {ISSUE_SUMMARY_FLAG}, None, IssueSummaryOutcome.DISABLED),
+ ("ai_not_approved", set(), lambda test: test._disapprove_ai(), IssueSummaryOutcome.AI_NOT_APPROVED),
+ (
+ "write_budget_drained",
+ set(),
+ lambda test: test._drain_write_budget(test.team.id),
+ IssueSummaryOutcome.RATE_LIMITED,
+ ),
+ ("all_checks_pass", set(), None, IssueSummaryOutcome.GENERATED),
+ ]
+ )
+ def test_issue_created_activity_fails_closed(
+ self, _name: str, flags_off: set[str], arrange: Any, expected: IssueSummaryOutcome
+ ) -> None:
+ self.enterContext(
+ patch.object(
+ team_flags, "feature_enabled_or_false", side_effect=lambda flag, *_, **__: flag not in flags_off
+ )
+ )
+ fetch = self.enterContext(
+ patch.object(issue_created_activities, "fetch_event_properties", return_value={"$lib": "web"})
+ )
+ capture = MagicMock()
+ self.enterContext(patch.object(issue_summaries, "ph_background_capture", return_value=capture))
+ if arrange is not None:
+ arrange(self)
+
+ outcome = generate_issue_created_summary_activity(self._issue_created_inputs())
+
+ assert outcome == expected.value
+ assert self.llm_calls == (1 if expected == IssueSummaryOutcome.GENERATED else 0)
+ assert fetch.called is (expected == IssueSummaryOutcome.GENERATED)
+ assert capture.called is (ISSUE_SUMMARY_ON_WRITE_FLAG not in flags_off)
+
+ def test_issue_created_activity_skips_while_worker_summary_slots_are_full(self) -> None:
+ self.enterContext(patch.object(team_flags, "feature_enabled_or_false", return_value=True))
+ self.enterContext(
+ patch.object(issue_created_activities, "fetch_event_properties", return_value={"$lib": "web"})
+ )
+ slots = threading.BoundedSemaphore(1)
+ self.enterContext(patch.object(issue_created_activities, "ISSUE_SUMMARY_WORKER_SLOTS", slots))
- assert self._generate_on_creation(budget_key=creation_key, budget=tiny) == IssueSummaryOutcome.RATE_LIMITED
+ slots.acquire()
+ assert (
+ generate_issue_created_summary_activity(self._issue_created_inputs())
+ == IssueSummaryOutcome.RATE_LIMITED.value
+ )
assert self.llm_calls == 0
+ slots.release()
+ assert (
+ generate_issue_created_summary_activity(self._issue_created_inputs()) == IssueSummaryOutcome.GENERATED.value
+ )
+ assert slots.acquire(blocking=False)
+
+ def _drain_write_budget(self, team_id: int) -> None:
+ tiny = Budget(burst=1, per_hour=1)
+ self.enterContext(patch.object(issue_summaries, "ISSUE_SUMMARY_WRITE_BUDGET", tiny))
+ consume(f"{issue_summaries.ISSUE_SUMMARY_WRITE_BUDGET_KEY_PREFIX}:{team_id}", tiny)
+
+ def _issue_created_inputs(self) -> IssueCreatedWorkflowInputs:
+ return IssueCreatedWorkflowInputs(
+ notification_id="notification",
+ team_id=self.team.id,
+ issue_id=str(self.issue.id),
+ issue=IssueCreatedSnapshot(
+ name="TypeError", description=None, status="active", created_at=NOW.isoformat(), severity=None
+ ),
+ fingerprint="fp",
+ event_uuid="00000000-0000-0000-0000-000000000001",
+ event_timestamp=NOW.isoformat(),
+ )
+
def test_merge_keeps_target_summary_and_deletes_source_summary(self) -> None:
source = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")
ErrorTrackingIssueFingerprintV2.objects.create(team=self.team, issue=source, fingerprint="fp-source")