:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(error-tracking): stop issue summary reads waiting for a summary that never comes (#122597)
James Gill59a2bea5bb29
diff --git a/products/error_tracking/backend/facade/contracts.py b/products/error_tracking/backend/facade/contracts.pyindex 41d748bb6aac..14a2939ab77c 100644--- a/products/error_tracking/backend/facade/contracts.py+++ b/products/error_tracking/backend/facade/contracts.py@@ -454,3 +454,5 @@ class ErrorTrackingIssueSummary:category: IssueSummaryCategory | Nonemodel: strgenerated_at: datetime+ # When the summary was last generated or confirmed unchanged. The 24h regenerate interval runs from it.+ checked_at: datetimediff --git a/products/error_tracking/backend/facade/summaries.py b/products/error_tracking/backend/facade/summaries.pyindex 0aeb8803b124..72e4ef22f745 100644--- a/products/error_tracking/backend/facade/summaries.py+++ b/products/error_tracking/backend/facade/summaries.py@@ -6,6 +6,8 @@ A module of its own, like ``queries.py``, so the HogQL and LLM client imports stfrom datetime import datetimefrom uuid import UUID+from posthog.models import User+from products.error_tracking.backend.facade import contractsfrom products.error_tracking.backend.logic import issue_summariesfrom products.error_tracking.backend.models import ErrorTrackingIssueSummary@@ -25,31 +27,50 @@ def _to_summary(summary: ErrorTrackingIssueSummary) -> contracts.ErrorTrackingIselse None,model=summary.model,generated_at=summary.generated_at,+ checked_at=summary.checked_at,)-def get_issue_summary(team_id: int, issue_id: UUID | str) -> contracts.ErrorTrackingIssueSummary | None:- """The stored summary, whatever its age. Never generates."""- summary = issue_summaries.get_issue_summary(team_id, issue_id)- return _to_summary(summary) if summary is not None else None+def _shown(team_id: int, summary: ErrorTrackingIssueSummary | None) -> contracts.ErrorTrackingIssueSummary | None:+ """Every read returns through here, so turning the flag off also hides stored summaries."""+ if summary is None or not issue_summaries.issue_summaries_enabled(team_id):+ return None+ return _to_summary(summary)+++def get_issue_summary(team_id: int, issue_id: UUID | str, *, user: User) -> contracts.ErrorTrackingIssueSummary | None:+ """The stored summary, whatever its age, when `user` may read it. Never generates."""+ if not issue_summaries.summary_sources_readable(team_id, user):+ return None+ return _shown(team_id, issue_summaries.get_issue_summary(team_id, issue_id))-def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> contracts.ErrorTrackingIssueSummary | None:+def get_or_generate_issue_summary(+ team_id: int, issue_id: UUID | str, *, user: User+) -> contracts.ErrorTrackingIssueSummary | None:"""Generate in the calling thread when a summary is due, else return the stored one. Blocks on the model."""- result = issue_summaries.get_or_generate_issue_summary(team_id, issue_id)- return _to_summary(result.summary) if result.summary is not None else None+ if not issue_summaries.summary_sources_readable(team_id, user):+ return None+ return _shown(team_id, issue_summaries.get_or_generate_issue_summary(team_id, issue_id).summary)++def start_issue_summary_generation(team_id: int, issue_id: UUID | str, *, user: User) -> bool:+ """Queue generation in the background when a summary may be due. False means the caller should not wait.-def start_issue_summary_generation(team_id: int, issue_id: UUID | str) -> bool:- """Queue generation in the background when a summary may be due. False means the caller should not wait."""+ Also False when `user` may not read the summary, so that caller does not wait for one it cannot see.+ """+ if not issue_summaries.summary_sources_readable(team_id, user):+ return Falsereturn issue_summaries.start_issue_summary_generation(team_id, issue_id)def wait_for_issue_summary(- team_id: int, issue_id: UUID | str, *, checked_after: datetime | None, timeout_seconds: float+ team_id: int, issue_id: UUID | str, *, user: User, checked_after: datetime | None, timeout_seconds: float) -> contracts.ErrorTrackingIssueSummary | None:"""Poll for a summary generated or confirmed unchanged after `checked_after`. None at the timeout."""+ if not issue_summaries.summary_sources_readable(team_id, user):+ return Nonesummary = issue_summaries.wait_for_issue_summary(team_id, issue_id, checked_after=checked_after, timeout_seconds=timeout_seconds)- return _to_summary(summary) if summary is not None else None+ return _shown(team_id, summary)diff --git a/products/error_tracking/backend/logic/issue_summaries.py b/products/error_tracking/backend/logic/issue_summaries.pyindex 087dddf02672..e32e0ab72474 100644--- a/products/error_tracking/backend/logic/issue_summaries.py+++ b/products/error_tracking/backend/logic/issue_summaries.py@@ -10,7 +10,7 @@ import timeimport hashlibfrom datetime import datetime, timedeltafrom enum import StrEnum-from uuid import UUID+from uuid import UUID, uuid4from django.db import IntegrityErrorfrom django.utils import timezone@@ -21,6 +21,7 @@ from openai.types.chat import ChatCompletionMessageParamfrom openai.types.shared_params import ResponseFormatJSONSchemafrom prometheus_client import Counterfrom pydantic import ValidationError+from redis import Redisfrom redis.exceptions import RedisErrorfrom posthog.schema import DateRange, ErrorTrackingQuery@@ -35,10 +36,15 @@ from posthog.dataclasses import frozenfrom posthog.exceptions_capture import capture_exceptionfrom posthog.llm.gateway_client import build_openai_client, resolve_ai_gateway_config, team_distinct_idfrom posthog.llm.semantic_enrichment import extract_json_object-from posthog.models import Team+from posthog.models import Team, Userfrom posthog.redis import get_clientfrom posthog.token_bucket import BucketDecision, BucketUnavailable, consume, peek, refund+from products.access_control.backend.facade.api import (+ get_restricted_properties_with_group_type_index_for_team,+ split_restricted_property_names,+)+from products.access_control.backend.facade.contracts import UserNotOrganizationMemberErrorfrom products.error_tracking.backend.facade.contracts import IssueSummaryCategoryfrom products.error_tracking.backend.facade.queries import run_error_tracking_queryfrom products.error_tracking.backend.facade.query_utils import (@@ -64,6 +70,8 @@ from products.error_tracking.backend.logic.issue_summary_config import (ISSUE_SUMMARY_MAX_OUTPUT_TOKENS,ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL,ISSUE_SUMMARY_MODEL,+ ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,+ ISSUE_SUMMARY_QUEUED_TTL_SECONDS,ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS,ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS,)@@ -104,7 +112,39 @@ IN_APP_FRAME_MARKER = "[in-app]"WAIT_POLL_INTERVAL_SECONDS = 0.25# Event values must not open or close the <issue> block that fences them in the prompt.ISSUE_FENCE_TAG = re.compile(r"</?\s*issue\s*>", re.IGNORECASE)-LOCK_HELD_FOR_GENERATION = "1"+# A worker claims with this prefix and its own token, so only that worker can settle the claim.+LOCK_CLAIM_PREFIX = "claim:"+# Workers deployed before claim tokens claim with this value.+LEGACY_LOCK_CLAIM = "1"+LOCK_QUEUED = "queued"+# Takes the lock when the key is free or holds only the queued marker. Returns nil on success, else the value that holds it.+CLAIM_LUA = """+local held = redis.call('GET', KEYS[1])+if held == false or held == ARGV[1] then+ redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3])+ return false+end+return held+"""+# Writes the outcome only while the key still holds the caller's claim. An empty TTL keeps the current one.+SETTLE_LUA = """+if redis.call('GET', KEYS[1]) ~= ARGV[1] then+ return 0+end+if ARGV[3] == '' then+ redis.call('SET', KEYS[1], ARGV[2], 'KEEPTTL')+else+ redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3])+end+return 1+"""+# Deletes the key only while it holds the queued marker, so a worker's claim or a hold outcome survives.+RELEASE_QUEUED_LUA = """+if redis.call('GET', KEYS[1]) == ARGV[1] then+ return redis.call('DEL', KEYS[1])+end+return 0+"""class IssueSummaryOutcome(StrEnum):@@ -123,8 +163,14 @@ class IssueSummaryOutcome(StrEnum):GATEWAY_UNCONFIGURED = "gateway_unconfigured"MODEL_ERROR = "model_error"INVALID_OUTPUT = "invalid_output"+ UNEXPECTED_ERROR = "unexpected_error"+# Outcomes that hold the issue past the lock TTL, because each try runs the input queries before it finds them.+HOLD_OUTCOMES = frozenset({IssueSummaryOutcome.NO_EXCEPTION, IssueSummaryOutcome.RATE_LIMITED})+# Outcomes that leave a summary current. The claim then stays until the lock expires.+SETTLED_OUTCOMES = frozenset({IssueSummaryOutcome.GENERATED, IssueSummaryOutcome.UNCHANGED, IssueSummaryOutcome.FRESH})+ISSUE_SUMMARY_OUTCOMES = Counter("error_tracking_issue_summary_total","Issue summary requests by outcome",@@ -149,6 +195,18 @@ def issue_summaries_enabled(team_id: int) -> bool:return team_flag_enabled(ISSUE_SUMMARY_FLAG, team_id)+def summary_sources_readable(team_id: int, user: User) -> bool:+ """False when the user may not read an event property the summary is written from.++ One summary serves the whole team, so a user with a property restriction could otherwise read it through the summary.+ """+ try:+ restrictions = get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team_id)+ except UserNotOrganizationMemberError:+ return False+ return ISSUE_SUMMARY_SOURCE_PROPERTIES.isdisjoint(split_restricted_property_names(restrictions).event)++def is_ai_data_processing_approved(team_id: int) -> bool:"""Fail closed when the team or its organization is missing."""return bool(@@ -177,16 +235,64 @@ def generation_blocked(team_id: int, stored: ErrorTrackingIssueSummary | None) -def start_issue_summary_generation(team_id: int, issue_id: UUID | str) -> bool:- """Queue generation in the background when a summary may be due. Returns whether it queued a task.+ """Queue generation in the background when a summary may be due. True when a summary may arrive soon.False means nothing will change soon, so the caller should not wait."""if generation_blocked(team_id, get_issue_summary(team_id, issue_id)) is not None:return False- generate_error_tracking_issue_summary.delay(team_id, str(issue_id))+ try:+ client = _start_client()+ key = _lock_key(team_id, issue_id)+ if client.set(key, LOCK_QUEUED, nx=True, ex=ISSUE_SUMMARY_QUEUED_TTL_SECONDS):+ return _queue_generation(team_id, issue_id)+ held = client.get(key)+ except RedisError:+ logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)+ return _queue_generation(team_id, issue_id)+ return _lock_outcome(held) == IssueSummaryOutcome.IN_PROGRESS+++def _queue_generation(team_id: int, issue_id: UUID | str) -> bool:+ try:+ _publish_generation(team_id, issue_id)+ except Exception as error:+ capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)})+ # Without a task, the queued marker would keep reads waiting until it expires.+ try:+ _start_client().register_script(RELEASE_QUEUED_LUA)(keys=[_lock_key(team_id, issue_id)], args=[LOCK_QUEUED])+ except RedisError:+ logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)+ return Falsereturn True+def _start_client() -> Redis:+ """The lock client for the start, with the publish timeouts.++ The lock and the broker share a Redis, so the default client's longer socket timeout would hold the read first.+ """+ return get_client(+ socket_timeout=ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,+ socket_connect_timeout=ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,+ )+++def _publish_generation(team_id: int, issue_id: UUID | str) -> None:+ """Queue the task on a connection with short timeouts and no publish retries.++ The shared producer pool has no socket timeout and retries the publish, so an unresponsive broker blocks the read.+ """+ timeout = ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS+ with generate_error_tracking_issue_summary.app.connection_for_write(+ connect_timeout=timeout,+ transport_options={"socket_timeout": timeout, "socket_connect_timeout": timeout},+ ) as connection:+ generate_error_tracking_issue_summary.apply_async(+ args=(team_id, str(issue_id)), connection=connection, retry=False+ )++def wait_for_issue_summary(team_id: int, issue_id: UUID | str, *, checked_after: datetime | None, timeout_seconds: float) -> ErrorTrackingIssueSummary | None:@@ -211,7 +317,11 @@ def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> IssueSuGeneration runs at most once at a time per issue. A failure keeps the stored summary."""- result = _get_or_generate(team_id, issue_id)+ try:+ result = _get_or_generate(team_id, issue_id)+ except Exception:+ ISSUE_SUMMARY_OUTCOMES.labels(outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value).inc()+ raiseISSUE_SUMMARY_OUTCOMES.labels(outcome=result.outcome.value).inc()logger.info("error_tracking_issue_summary",@@ -227,9 +337,20 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:stored = get_issue_summary(team_id, issue_id)if (blocked := generation_blocked(team_id, stored)) is not None:return IssueSummaryResult(outcome=blocked, summary=stored)- if (held := _claim_generation(team_id, issue_id)) is not None:+ claim = f"{LOCK_CLAIM_PREFIX}{uuid4().hex}"+ if (held := _claim_generation(team_id, issue_id, claim)) is not None:return IssueSummaryResult(outcome=held, summary=stored)+ try:+ result = _generate_claimed(team_id, issue_id)+ except Exception:+ _settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR)+ raise+ _settle_claim(team_id, issue_id, claim, result.outcome)+ return result+++def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:# Another worker can finish between the first read and the claim.stored = get_issue_summary(team_id, issue_id)if not regeneration_due(stored, timezone.now()):@@ -239,7 +360,6 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:# The input queries run before `consume`, so an unchanged issue spends no token. This read keeps a team# with no tokens left from running them for every issue it opens. `consume` still makes the decision.if _budget_exhausted(budget_key):- _hold_generation(team_id, issue_id, IssueSummaryOutcome.RATE_LIMITED)return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)team = Team.objects.get(id=team_id)@@ -250,8 +370,6 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:logger.warning("error_tracking_issue_summary_inputs_failed", team_id=team_id, issue_id=str(issue_id))return IssueSummaryResult(outcome=IssueSummaryOutcome.INPUTS_ERROR, summary=stored)if isinstance(inputs, IssueSummaryOutcome):- if inputs == IssueSummaryOutcome.NO_EXCEPTION:- _hold_generation(team_id, issue_id, inputs)return IssueSummaryResult(outcome=inputs, summary=stored)if stored is not None and stored.input_hash == inputs.input_hash:stored.checked_at = timezone.now()@@ -261,7 +379,6 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:decision = consume(budget_key, ISSUE_SUMMARY_BUDGET)match decision:case BucketDecision(allowed=False):- _hold_generation(team_id, issue_id, IssueSummaryOutcome.RATE_LIMITED)return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)case BucketUnavailable(error=bucket_error):# Fail open: the per-issue lock and the regenerate interval still bound the spend.@@ -313,28 +430,39 @@ def _lock_key(team_id: int, issue_id: UUID | str) -> str:return f"{ISSUE_SUMMARY_LOCK_KEY_PREFIX}:{team_id}:{issue_id}"-def _claim_generation(team_id: int, issue_id: UUID | str) -> IssueSummaryOutcome | None:- """None when this worker now holds the issue, else the outcome that holds it."""+def _claim_generation(team_id: int, issue_id: UUID | str, claim: str) -> IssueSummaryOutcome | None:+ """None when this worker now holds the issue under `claim`, else the outcome that holds it."""try:- client = get_client()- key = _lock_key(team_id, issue_id)- if client.set(key, LOCK_HELD_FOR_GENERATION, nx=True, ex=ISSUE_SUMMARY_LOCK_TTL_SECONDS):- return None- held = client.get(key)+ held = get_client().register_script(CLAIM_LUA)(+ keys=[_lock_key(team_id, issue_id)],+ args=[LOCK_QUEUED, claim, ISSUE_SUMMARY_LOCK_TTL_SECONDS],+ )except RedisError:# Fail open, as the budget does: a Redis outage can let two concurrent reads both pay.logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)return None+ return _lock_outcome(held) if held is not None else None+++def _lock_outcome(held: bytes | str | None) -> IssueSummaryOutcome:+ """A live claim or a queued task means a summary may come. Every other value is the outcome that stopped generation."""value = held.decode() if isinstance(held, bytes) else held- if value in (IssueSummaryOutcome.NO_EXCEPTION, IssueSummaryOutcome.RATE_LIMITED):- return IssueSummaryOutcome(value)- return IssueSummaryOutcome.IN_PROGRESS+ if value is None or value in (LEGACY_LOCK_CLAIM, LOCK_QUEUED) or value.startswith(LOCK_CLAIM_PREFIX):+ return IssueSummaryOutcome.IN_PROGRESS+ return IssueSummaryOutcome(value)++def _settle_claim(team_id: int, issue_id: UUID | str, claim: str, outcome: IssueSummaryOutcome) -> None:+ """Replace this worker's claim with `outcome` when no summary came of it, so reads stop waiting for one.-def _hold_generation(team_id: int, issue_id: UUID | str, outcome: IssueSummaryOutcome) -> None:- """Keep the issue claimed past the lock TTL, so reads return `outcome` without running the queries again."""+ A hold outcome extends the lock. Any other failure keeps the lock TTL, so the retry spacing does not change.+ Nothing is written when the key no longer holds `claim`: the lock expired, or another read took the issue.+ """+ if outcome in SETTLED_OUTCOMES:+ return+ ttl = str(ISSUE_SUMMARY_HOLD_SECONDS) if outcome in HOLD_OUTCOMES else ""try:- get_client().set(_lock_key(team_id, issue_id), outcome.value, ex=ISSUE_SUMMARY_HOLD_SECONDS)+ get_client().register_script(SETTLE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[claim, outcome.value, ttl])except RedisError:logger.warning("error_tracking_issue_summary_hold_unavailable", team_id=team_id, exc_info=True)@@ -350,6 +478,13 @@ def compute_input_hash(*, fingerprints: list[str], latest_release: str | None, oreturn hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()+# The only event properties the prompt builder sees, and the ones `summary_sources_readable` checks.+# `_exception_event_properties` drops every other key, so a property left out is missing from the prompt, not leaked.+ISSUE_SUMMARY_SOURCE_PROPERTIES = frozenset(+ {"$exception_list", "$exception_level", "$lib", "$current_url", "$exception_release"}+)++def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummaryInputs | IssueSummaryOutcome:"""Build the prompt and the input hash, or say why the issue cannot be summarised."""issue = ErrorTrackingIssue.objects.with_first_seen().filter(team_id=team.id, id=issue_id).first()@@ -496,7 +631,8 @@ def _exception_event_properties()if not response.results:return {}- return as_record(parse_jsonish(response.results[0][0])) or {}+ properties = as_record(parse_jsonish(response.results[0][0])) or {}+ return {key: value for key, value in properties.items() if key in ISSUE_SUMMARY_SOURCE_PROPERTIES}def _aggregated_issue(team: Team, issue_id: UUID | str) -> dict[str, object]:diff --git a/products/error_tracking/backend/logic/issue_summary_config.py b/products/error_tracking/backend/logic/issue_summary_config.pyindex 8384cd972727..b69c01c074b6 100644--- a/products/error_tracking/backend/logic/issue_summary_config.py+++ b/products/error_tracking/backend/logic/issue_summary_config.py@@ -25,6 +25,11 @@ ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL = timedelta(hours=24)# Holds one generation per issue at a time. It is never released, so it also spaces out retries after a failure.ISSUE_SUMMARY_LOCK_TTL_SECONDS = 300+# Marks a queued task until a worker claims it, so page polls queue at most one task per issue in this window.+# A lost task keeps reads on "generating" for this long, so it stays short.+ISSUE_SUMMARY_QUEUED_TTL_SECONDS = 60+# Bounds the lock calls and the publish when a read queues a task, so a slow Redis cannot hold the request.+ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS = 1.0# An issue with no recent exception event, or a team out of budget, is not tried again for this long.# Each try runs the ClickHouse queries before it can find either outcome.ISSUE_SUMMARY_HOLD_SECONDS = 3600diff --git a/products/error_tracking/backend/tests/test_issue_summaries.py b/products/error_tracking/backend/tests/test_issue_summaries.pyindex eae757f021b6..ab15ae21d127 100644--- a/products/error_tracking/backend/tests/test_issue_summaries.py+++ b/products/error_tracking/backend/tests/test_issue_summaries.py@@ -1,4 +1,6 @@import json+import signal+import socketfrom datetime import UTC, datetime, timedeltafrom typing import Any@@ -6,14 +8,19 @@ import time_machinefrom posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_eventsfrom unittest.mock import MagicMock, patch+from django.db import OperationalErrorfrom django.test import SimpleTestCasefrom parameterized import parameterizedfrom redis.exceptions import RedisError+from posthog.constants import AvailableFeature+from posthog.models import PropertyDefinition, Userfrom posthog.redis import TEST_clear_clients-from posthog.token_bucket import BucketUnavailable, Budget, TEST_reset_scripts, consume+from posthog.token_bucket import BucketUnavailable, Budget, TEST_reset_scripts, consume, refund+from products.access_control.backend.facade.api import upsert_property_access_control+from products.access_control.backend.facade.contracts import PropertyAccessLevel, UpsertPropertyAccessControlInputfrom products.error_tracking.backend.facade import summaries as summaries_facadefrom products.error_tracking.backend.facade.contracts import IssueSummaryCategoryfrom products.error_tracking.backend.logic import issue_summaries, team_flags@@ -91,6 +98,11 @@ FAILURE_CASES: list[tuple[str, Any, IssueSummaryOutcome]] = [IssueSummaryOutcome.INPUTS_ERROR,),("cap_denied", lambda test: test._drain_budget(), IssueSummaryOutcome.RATE_LIMITED),+ (+ "gateway_unconfigured",+ lambda test: test.enterContext(patch.object(issue_summaries, "resolve_ai_gateway_config", return_value=None)),+ IssueSummaryOutcome.GATEWAY_UNCONFIGURED,+ ),("llm_raises",lambda test: test.completions.configure_mock(side_effect=TimeoutError("slow")),@@ -137,10 +149,12 @@ class TestIssueSummaryGeneration(BaseTest):def llm_calls(self) -> int:return self.completions.call_count+ @property+ def _lock_key(self) -> str:+ return issue_summaries._lock_key(self.team.id, self.issue.id)+def _release_lock(self) -> None:- issue_summaries.get_client().delete(- f"{issue_summaries.ISSUE_SUMMARY_LOCK_KEY_PREFIX}:{self.team.id}:{self.issue.id}"- )+ issue_summaries.get_client().delete(self._lock_key)@parameterized.expand([@@ -157,7 +171,7 @@ class TestIssueSummaryGeneration(BaseTest):result = get_or_generate_issue_summary(self.team.id, self.issue.id)assert result.outcome == IssueSummaryOutcome.GENERATED- summary = summaries_facade.get_issue_summary(self.team.id, self.issue.id)+ summary = summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=self.user)assert summary is not Noneassert summary.what_happened == SUMMARY_REPLY["what_happened"]assert summary.category == expected_category@@ -179,10 +193,17 @@ class TestIssueSummaryGeneration(BaseTest):assert result.outcome == IssueSummaryOutcome.GENERATED+ @property+ def _budget_key(self) -> str:+ return f"{issue_summaries.ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{self.team.id}"+def _drain_budget(self) -> None:tiny = Budget(burst=1, per_hour=1)self.enterContext(patch.object(issue_summaries, "ISSUE_SUMMARY_BUDGET", tiny))- consume(f"{issue_summaries.ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{self.team.id}", tiny)+ consume(self._budget_key, tiny)++ def _refill_budget(self) -> None:+ refund(self._budget_key, issue_summaries.ISSUE_SUMMARY_BUDGET)def _disapprove_ai(self) -> None:self.organization.is_ai_data_processing_approved = False@@ -198,13 +219,6 @@ class TestIssueSummaryGeneration(BaseTest):IssueSummaryOutcome.DISABLED,),("ai_not_approved", lambda test: test._disapprove_ai(), IssueSummaryOutcome.AI_NOT_APPROVED),- (- "gateway_unconfigured",- lambda test: test.enterContext(- patch.object(issue_summaries, "resolve_ai_gateway_config", return_value=None)- ),- IssueSummaryOutcome.GATEWAY_UNCONFIGURED,- ),*FAILURE_CASES,])@@ -251,6 +265,7 @@ class TestIssueSummaryGeneration(BaseTest):arrange(self)with time_machine.travel(NOW, tick=False):get_or_generate_issue_summary(self.team.id, self.issue.id)+ self._refill_budget()with time_machine.travel(NOW + timedelta(minutes=10), tick=False):result = get_or_generate_issue_summary(self.team.id, self.issue.id)@@ -289,16 +304,19 @@ class TestIssueSummaryGeneration(BaseTest):with (time_machine.travel(NOW + later, tick=False),- patch.object(issue_summaries.generate_error_tracking_issue_summary, "delay") as delay,+ patch.object(issue_summaries.generate_error_tracking_issue_summary, "apply_async") as apply_async,):result = get_or_generate_issue_summary(self.team.id, self.issue.id)- assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id) is False- delay.assert_not_called()+ assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user) is False+ apply_async.assert_not_called()assert result.outcome == expectedassert result.summary is not Noneassert self.llm_calls == (2 if expected == IssueSummaryOutcome.GENERATED else 1)assert result.summary.input_hash == (new_hash if expected == IssueSummaryOutcome.GENERATED else "hash-1")+ shown = summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=self.user)+ assert shown is not None+ assert (shown.generated_at, shown.checked_at) == (result.summary.generated_at, result.summary.checked_at)def test_merge_keeps_target_summary_and_deletes_source_summary(self) -> None:source = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")@@ -313,19 +331,202 @@ class TestIssueSummaryGeneration(BaseTest):)assert stored_issue_ids == {self.issue.id}+ def _start(self) -> tuple[bool, bool]:+ with patch.object(issue_summaries.generate_error_tracking_issue_summary, "apply_async") as apply_async:+ started = summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user)+ return started, apply_async.called++ @parameterized.expand(+ [+ ("due", True, None, True, True),+ ("flag_off", False, None, False, False),+ ("held_no_exception", True, IssueSummaryOutcome.NO_EXCEPTION.value, False, False),+ ("held_rate_limited", True, IssueSummaryOutcome.RATE_LIMITED.value, False, False),+ ("held_after_failure", True, IssueSummaryOutcome.MODEL_ERROR.value, False, False),+ ("claimed_by_another_worker", True, f"{issue_summaries.LOCK_CLAIM_PREFIX}other", True, False),+ ("claimed_by_an_older_worker", True, issue_summaries.LEGACY_LOCK_CLAIM, True, False),+ ]+ )+ def test_start_generation_queues_only_when_it_can_generate(+ self, _name: str, flag: bool, lock_value: str | None, started: bool, queued: bool+ ) -> None:+ if lock_value is not None:+ issue_summaries.get_client().set(self._lock_key, lock_value, ex=60)++ with patch.object(team_flags, "feature_enabled_or_false", return_value=flag):+ assert self._start() == (started, queued)++ def test_start_generation_queues_when_redis_is_down(self) -> None:+ with patch.object(issue_summaries, "get_client", side_effect=RedisError("down")):+ assert self._start() == (True, True)++ def test_start_queues_one_task_until_a_worker_claims_it(self) -> None:+ assert self._start() == (True, True)+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= issue_summaries.ISSUE_SUMMARY_QUEUED_TTL_SECONDS+ assert self._start() == (True, False)++ result = get_or_generate_issue_summary(self.team.id, self.issue.id)++ assert result.outcome == IssueSummaryOutcome.GENERATED++ def test_start_returns_false_when_the_broker_is_down(self) -> None:+ capture = self.enterContext(patch.object(issue_summaries, "capture_exception"))+ with patch.object(+ issue_summaries.generate_error_tracking_issue_summary, "apply_async", side_effect=OSError("broker down")+ ):+ assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user) is False++ capture.assert_called_once()+ assert self._start() == (True, True)+@parameterized.expand([- ("due", True, True),- ("flag_off", False, False),+ ("llm_raises", lambda test: test.completions.configure_mock(side_effect=TimeoutError("slow"))),+ (+ "issue_missing",+ lambda test: test.build_inputs.configure_mock(+ side_effect=None, return_value=IssueSummaryOutcome.ISSUE_MISSING+ ),+ ),])- def test_start_generation_queues_only_when_it_can_generate(self, _name: str, flag: bool, queued: bool) -> None:+ def test_failure_stops_start_until_the_lock_expires(self, _name: str, arrange: Any) -> None:+ arrange(self)+ get_or_generate_issue_summary(self.team.id, self.issue.id)++ assert self._start() == (False, False)+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= issue_summaries.ISSUE_SUMMARY_LOCK_TTL_SECONDS++ self._release_lock()+ assert self._start() == (True, True)++ @parameterized.expand(+ [+ ("failure", "completions", TimeoutError("slow"), IssueSummaryOutcome.MODEL_ERROR),+ ("hold", "build_inputs", IssueSummaryOutcome.NO_EXCEPTION, IssueSummaryOutcome.NO_EXCEPTION),+ ]+ )+ def test_settle_does_not_overwrite_a_lock_this_worker_no_longer_holds(+ self, _name: str, mock_name: str, result_after: Any, expected: IssueSummaryOutcome+ ) -> None:+ def lose_the_lock(*_args: Any, **_kwargs: Any) -> Any:+ issue_summaries.get_client().set(self._lock_key, issue_summaries.LOCK_QUEUED, ex=60)+ if isinstance(result_after, Exception):+ raise result_after+ return result_after++ getattr(self, mock_name).side_effect = lose_the_lock++ result = get_or_generate_issue_summary(self.team.id, self.issue.id)++ assert result.outcome == expected+ assert issue_summaries.get_client().get(self._lock_key) == issue_summaries.LOCK_QUEUED.encode()+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= 60++ def test_unexpected_error_stops_start_and_still_raises(self) -> None:with (- patch.object(team_flags, "feature_enabled_or_false", return_value=flag),- patch.object(issue_summaries.generate_error_tracking_issue_summary, "delay") as delay,+ patch.object(issue_summaries.Team.objects, "get", side_effect=OperationalError("db down")),+ self.assertRaises(OperationalError),):- assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id) == queued- assert delay.called == queued+ get_or_generate_issue_summary(self.team.id, self.issue.id)++ assert self._start() == (False, False)+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= issue_summaries.ISSUE_SUMMARY_LOCK_TTL_SECONDS++ @parameterized.expand(+ [+ ("get", lambda test: summaries_facade.get_issue_summary(test.team.id, test.issue.id, user=test.user)),+ (+ "wait",+ lambda test: summaries_facade.wait_for_issue_summary(+ test.team.id, test.issue.id, user=test.user, checked_after=None, timeout_seconds=0+ ),+ ),+ (+ "get_or_generate",+ lambda test: summaries_facade.get_or_generate_issue_summary(+ test.team.id, test.issue.id, user=test.user+ ),+ ),+ ]+ )+ def test_flag_off_or_restricted_source_hides_stored_summary(self, _name: str, read: Any) -> None:+ get_or_generate_issue_summary(self.team.id, self.issue.id)+ assert read(self) is not None++ with patch.object(team_flags, "feature_enabled_or_false", return_value=False):+ assert read(self) is None++ self._restrict_event_property("$current_url")+ assert read(self) is None++ def _restrict_event_property(self, name: str) -> None:+ self.organization.available_product_features = [+ {"name": AvailableFeature.PROPERTY_ACCESS_CONTROL, "key": AvailableFeature.PROPERTY_ACCESS_CONTROL}+ ]+ self.organization.save()+ definition = PropertyDefinition.objects.create(team=self.team, name=name, type=PropertyDefinition.Type.EVENT)+ upsert_property_access_control(+ team_id=self.team.id,+ created_by_id=self.user.id,+ input=UpsertPropertyAccessControlInput(+ property_definition_id=str(definition.id),+ access_level=PropertyAccessLevel.NONE,+ organization_member_id=self.organization_membership.id,+ ),+ )++ def test_restricted_member_cannot_start_and_other_members_still_read(self) -> None:+ other = User.objects.create_and_join(self.organization, "[email protected]", "password")+ self._restrict_event_property("$exception_list")++ assert self._start() == (False, False)+ assert summaries_facade.get_or_generate_issue_summary(self.team.id, self.issue.id, user=self.user) is None+ assert self.llm_calls == 0++ get_or_generate_issue_summary(self.team.id, self.issue.id)+ assert summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=other) is not None+ outsider = User.objects.create(email="[email protected]")+ assert summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=outsider) is None++ def test_start_gives_up_quickly_when_the_broker_does_not_answer(self) -> None:+ # Not an Exception, because `_queue_generation` catches Exception and would turn a hung publish into a pass.+ class StartBlocked(BaseException):+ pass++ def reset_producer_pool() -> None:+ app._pool = None+ app.amqp._producer_pool = None++ self.enterContext(patch.object(issue_summaries, "ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS", 0.05))+ silent_broker = socket.socket()+ silent_broker.bind(("127.0.0.1", 0))+ silent_broker.listen()+ self.addCleanup(silent_broker.close)+ app = issue_summaries.generate_error_tracking_issue_summary.app+ silent_url = f"redis://127.0.0.1:{silent_broker.getsockname()[1]}/0"+ # Settings name the broker with the old CELERY_ key, which wins over `broker_url`.+ saved = {key: app.conf[key] for key in ("task_always_eager", "CELERY_BROKER_URL")}+ # The producer pool keeps the broker it first connected to, so drop it on both sides of the swap.+ self.addCleanup(reset_producer_pool)+ self.addCleanup(app.conf.update, saved)+ app.conf.update(task_always_eager=False, CELERY_BROKER_URL=silent_url)+ reset_producer_pool()+ assert app.connection_for_write().as_uri() == silent_url+ self.enterContext(patch.object(issue_summaries, "capture_exception"))++ def hung(*_args: Any) -> None:+ raise StartBlocked++ previous = signal.signal(signal.SIGALRM, hung)+ self.addCleanup(signal.signal, signal.SIGALRM, previous)+ signal.alarm(3)+ try:+ started = summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user)+ finally:+ signal.alarm(0)++ assert started is False@parameterized.expand([@@ -341,7 +542,7 @@ class TestIssueSummaryGeneration(BaseTest):get_or_generate_issue_summary(self.team.id, self.issue.id)summary = summaries_facade.wait_for_issue_summary(- self.team.id, self.issue.id, checked_after=checked_after, timeout_seconds=0+ self.team.id, self.issue.id, user=self.user, checked_after=checked_after, timeout_seconds=0)assert (summary is not None) == found