Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(error-tracking): stop issue summary reads waiting for a summary that never comes (#122597)

James Gill59a2bea5bb29
diff --git a/products/error_tracking/backend/facade/contracts.py b/products/error_tracking/backend/facade/contracts.py
index 41d748bb6aac..14a2939ab77c 100644
--- a/products/error_tracking/backend/facade/contracts.py
+++ b/products/error_tracking/backend/facade/contracts.py
@@ -454,3 +454,5 @@ class ErrorTrackingIssueSummary:
category: IssueSummaryCategory | None
model: str
generated_at: datetime
+ # When the summary was last generated or confirmed unchanged. The 24h regenerate interval runs from it.
+ checked_at: datetime
diff --git a/products/error_tracking/backend/facade/summaries.py b/products/error_tracking/backend/facade/summaries.py
index 0aeb8803b124..72e4ef22f745 100644
--- a/products/error_tracking/backend/facade/summaries.py
+++ b/products/error_tracking/backend/facade/summaries.py
@@ -6,6 +6,8 @@ A module of its own, like ``queries.py``, so the HogQL and LLM client imports st
from datetime import datetime
from uuid import UUID
+from posthog.models import User
+
from products.error_tracking.backend.facade import contracts
from products.error_tracking.backend.logic import issue_summaries
from products.error_tracking.backend.models import ErrorTrackingIssueSummary
@@ -25,31 +27,50 @@ def _to_summary(summary: ErrorTrackingIssueSummary) -> contracts.ErrorTrackingIs
else None,
model=summary.model,
generated_at=summary.generated_at,
+ checked_at=summary.checked_at,
)
-def get_issue_summary(team_id: int, issue_id: UUID | str) -> contracts.ErrorTrackingIssueSummary | None:
- """The stored summary, whatever its age. Never generates."""
- summary = issue_summaries.get_issue_summary(team_id, issue_id)
- return _to_summary(summary) if summary is not None else None
+def _shown(team_id: int, summary: ErrorTrackingIssueSummary | None) -> contracts.ErrorTrackingIssueSummary | None:
+ """Every read returns through here, so turning the flag off also hides stored summaries."""
+ if summary is None or not issue_summaries.issue_summaries_enabled(team_id):
+ return None
+ return _to_summary(summary)
+
+
+def get_issue_summary(team_id: int, issue_id: UUID | str, *, user: User) -> contracts.ErrorTrackingIssueSummary | None:
+ """The stored summary, whatever its age, when `user` may read it. Never generates."""
+ if not issue_summaries.summary_sources_readable(team_id, user):
+ return None
+ return _shown(team_id, issue_summaries.get_issue_summary(team_id, issue_id))
-def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> contracts.ErrorTrackingIssueSummary | None:
+def get_or_generate_issue_summary(
+ team_id: int, issue_id: UUID | str, *, user: User
+) -> contracts.ErrorTrackingIssueSummary | None:
"""Generate in the calling thread when a summary is due, else return the stored one. Blocks on the model."""
- result = issue_summaries.get_or_generate_issue_summary(team_id, issue_id)
- return _to_summary(result.summary) if result.summary is not None else None
+ if not issue_summaries.summary_sources_readable(team_id, user):
+ return None
+ return _shown(team_id, issue_summaries.get_or_generate_issue_summary(team_id, issue_id).summary)
+
+def start_issue_summary_generation(team_id: int, issue_id: UUID | str, *, user: User) -> bool:
+ """Queue generation in the background when a summary may be due. False means the caller should not wait.
-def start_issue_summary_generation(team_id: int, issue_id: UUID | str) -> bool:
- """Queue generation in the background when a summary may be due. False means the caller should not wait."""
+ Also False when `user` may not read the summary, so that caller does not wait for one it cannot see.
+ """
+ if not issue_summaries.summary_sources_readable(team_id, user):
+ return False
return issue_summaries.start_issue_summary_generation(team_id, issue_id)
def wait_for_issue_summary(
- team_id: int, issue_id: UUID | str, *, checked_after: datetime | None, timeout_seconds: float
+ team_id: int, issue_id: UUID | str, *, user: User, checked_after: datetime | None, timeout_seconds: float
) -> contracts.ErrorTrackingIssueSummary | None:
"""Poll for a summary generated or confirmed unchanged after `checked_after`. None at the timeout."""
+ if not issue_summaries.summary_sources_readable(team_id, user):
+ return None
summary = issue_summaries.wait_for_issue_summary(
team_id, issue_id, checked_after=checked_after, timeout_seconds=timeout_seconds
)
- return _to_summary(summary) if summary is not None else None
+ return _shown(team_id, summary)
diff --git a/products/error_tracking/backend/logic/issue_summaries.py b/products/error_tracking/backend/logic/issue_summaries.py
index 087dddf02672..e32e0ab72474 100644
--- a/products/error_tracking/backend/logic/issue_summaries.py
+++ b/products/error_tracking/backend/logic/issue_summaries.py
@@ -10,7 +10,7 @@ import time
import hashlib
from datetime import datetime, timedelta
from enum import StrEnum
-from uuid import UUID
+from uuid import UUID, uuid4
from django.db import IntegrityError
from django.utils import timezone
@@ -21,6 +21,7 @@ from openai.types.chat import ChatCompletionMessageParam
from openai.types.shared_params import ResponseFormatJSONSchema
from prometheus_client import Counter
from pydantic import ValidationError
+from redis import Redis
from redis.exceptions import RedisError
from posthog.schema import DateRange, ErrorTrackingQuery
@@ -35,10 +36,15 @@ from posthog.dataclasses import frozen
from posthog.exceptions_capture import capture_exception
from posthog.llm.gateway_client import build_openai_client, resolve_ai_gateway_config, team_distinct_id
from posthog.llm.semantic_enrichment import extract_json_object
-from posthog.models import Team
+from posthog.models import Team, User
from posthog.redis import get_client
from posthog.token_bucket import BucketDecision, BucketUnavailable, consume, peek, refund
+from products.access_control.backend.facade.api import (
+ get_restricted_properties_with_group_type_index_for_team,
+ split_restricted_property_names,
+)
+from products.access_control.backend.facade.contracts import UserNotOrganizationMemberError
from products.error_tracking.backend.facade.contracts import IssueSummaryCategory
from products.error_tracking.backend.facade.queries import run_error_tracking_query
from products.error_tracking.backend.facade.query_utils import (
@@ -64,6 +70,8 @@ from products.error_tracking.backend.logic.issue_summary_config import (
ISSUE_SUMMARY_MAX_OUTPUT_TOKENS,
ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL,
ISSUE_SUMMARY_MODEL,
+ ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,
+ ISSUE_SUMMARY_QUEUED_TTL_SECONDS,
ISSUE_SUMMARY_RECENT_EVENT_LOOKBACK_DAYS,
ISSUE_SUMMARY_STACKTRACE_MAX_TOKENS,
)
@@ -104,7 +112,39 @@ IN_APP_FRAME_MARKER = "[in-app]"
WAIT_POLL_INTERVAL_SECONDS = 0.25
# Event values must not open or close the <issue> block that fences them in the prompt.
ISSUE_FENCE_TAG = re.compile(r"</?\s*issue\s*>", re.IGNORECASE)
-LOCK_HELD_FOR_GENERATION = "1"
+# A worker claims with this prefix and its own token, so only that worker can settle the claim.
+LOCK_CLAIM_PREFIX = "claim:"
+# Workers deployed before claim tokens claim with this value.
+LEGACY_LOCK_CLAIM = "1"
+LOCK_QUEUED = "queued"
+# Takes the lock when the key is free or holds only the queued marker. Returns nil on success, else the value that holds it.
+CLAIM_LUA = """
+local held = redis.call('GET', KEYS[1])
+if held == false or held == ARGV[1] then
+ redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3])
+ return false
+end
+return held
+"""
+# Writes the outcome only while the key still holds the caller's claim. An empty TTL keeps the current one.
+SETTLE_LUA = """
+if redis.call('GET', KEYS[1]) ~= ARGV[1] then
+ return 0
+end
+if ARGV[3] == '' then
+ redis.call('SET', KEYS[1], ARGV[2], 'KEEPTTL')
+else
+ redis.call('SET', KEYS[1], ARGV[2], 'EX', ARGV[3])
+end
+return 1
+"""
+# Deletes the key only while it holds the queued marker, so a worker's claim or a hold outcome survives.
+RELEASE_QUEUED_LUA = """
+if redis.call('GET', KEYS[1]) == ARGV[1] then
+ return redis.call('DEL', KEYS[1])
+end
+return 0
+"""
class IssueSummaryOutcome(StrEnum):
@@ -123,8 +163,14 @@ class IssueSummaryOutcome(StrEnum):
GATEWAY_UNCONFIGURED = "gateway_unconfigured"
MODEL_ERROR = "model_error"
INVALID_OUTPUT = "invalid_output"
+ UNEXPECTED_ERROR = "unexpected_error"
+# Outcomes that hold the issue past the lock TTL, because each try runs the input queries before it finds them.
+HOLD_OUTCOMES = frozenset({IssueSummaryOutcome.NO_EXCEPTION, IssueSummaryOutcome.RATE_LIMITED})
+# Outcomes that leave a summary current. The claim then stays until the lock expires.
+SETTLED_OUTCOMES = frozenset({IssueSummaryOutcome.GENERATED, IssueSummaryOutcome.UNCHANGED, IssueSummaryOutcome.FRESH})
+
ISSUE_SUMMARY_OUTCOMES = Counter(
"error_tracking_issue_summary_total",
"Issue summary requests by outcome",
@@ -149,6 +195,18 @@ def issue_summaries_enabled(team_id: int) -> bool:
return team_flag_enabled(ISSUE_SUMMARY_FLAG, team_id)
+def summary_sources_readable(team_id: int, user: User) -> bool:
+ """False when the user may not read an event property the summary is written from.
+
+ One summary serves the whole team, so a user with a property restriction could otherwise read it through the summary.
+ """
+ try:
+ restrictions = get_restricted_properties_with_group_type_index_for_team(user=user, team_id=team_id)
+ except UserNotOrganizationMemberError:
+ return False
+ return ISSUE_SUMMARY_SOURCE_PROPERTIES.isdisjoint(split_restricted_property_names(restrictions).event)
+
+
def is_ai_data_processing_approved(team_id: int) -> bool:
"""Fail closed when the team or its organization is missing."""
return bool(
@@ -177,16 +235,64 @@ def generation_blocked(team_id: int, stored: ErrorTrackingIssueSummary | None) -
def start_issue_summary_generation(team_id: int, issue_id: UUID | str) -> bool:
- """Queue generation in the background when a summary may be due. Returns whether it queued a task.
+ """Queue generation in the background when a summary may be due. True when a summary may arrive soon.
False means nothing will change soon, so the caller should not wait.
"""
if generation_blocked(team_id, get_issue_summary(team_id, issue_id)) is not None:
return False
- generate_error_tracking_issue_summary.delay(team_id, str(issue_id))
+ try:
+ client = _start_client()
+ key = _lock_key(team_id, issue_id)
+ if client.set(key, LOCK_QUEUED, nx=True, ex=ISSUE_SUMMARY_QUEUED_TTL_SECONDS):
+ return _queue_generation(team_id, issue_id)
+ held = client.get(key)
+ except RedisError:
+ logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)
+ return _queue_generation(team_id, issue_id)
+ return _lock_outcome(held) == IssueSummaryOutcome.IN_PROGRESS
+
+
+def _queue_generation(team_id: int, issue_id: UUID | str) -> bool:
+ try:
+ _publish_generation(team_id, issue_id)
+ except Exception as error:
+ capture_exception(error, {"team_id": team_id, "issue_id": str(issue_id)})
+ # Without a task, the queued marker would keep reads waiting until it expires.
+ try:
+ _start_client().register_script(RELEASE_QUEUED_LUA)(keys=[_lock_key(team_id, issue_id)], args=[LOCK_QUEUED])
+ except RedisError:
+ logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)
+ return False
return True
+def _start_client() -> Redis:
+ """The lock client for the start, with the publish timeouts.
+
+ The lock and the broker share a Redis, so the default client's longer socket timeout would hold the read first.
+ """
+ return get_client(
+ socket_timeout=ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,
+ socket_connect_timeout=ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS,
+ )
+
+
+def _publish_generation(team_id: int, issue_id: UUID | str) -> None:
+ """Queue the task on a connection with short timeouts and no publish retries.
+
+ The shared producer pool has no socket timeout and retries the publish, so an unresponsive broker blocks the read.
+ """
+ timeout = ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS
+ with generate_error_tracking_issue_summary.app.connection_for_write(
+ connect_timeout=timeout,
+ transport_options={"socket_timeout": timeout, "socket_connect_timeout": timeout},
+ ) as connection:
+ generate_error_tracking_issue_summary.apply_async(
+ args=(team_id, str(issue_id)), connection=connection, retry=False
+ )
+
+
def wait_for_issue_summary(
team_id: int, issue_id: UUID | str, *, checked_after: datetime | None, timeout_seconds: float
) -> ErrorTrackingIssueSummary | None:
@@ -211,7 +317,11 @@ def get_or_generate_issue_summary(team_id: int, issue_id: UUID | str) -> IssueSu
Generation runs at most once at a time per issue. A failure keeps the stored summary.
"""
- result = _get_or_generate(team_id, issue_id)
+ try:
+ result = _get_or_generate(team_id, issue_id)
+ except Exception:
+ ISSUE_SUMMARY_OUTCOMES.labels(outcome=IssueSummaryOutcome.UNEXPECTED_ERROR.value).inc()
+ raise
ISSUE_SUMMARY_OUTCOMES.labels(outcome=result.outcome.value).inc()
logger.info(
"error_tracking_issue_summary",
@@ -227,9 +337,20 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:
stored = get_issue_summary(team_id, issue_id)
if (blocked := generation_blocked(team_id, stored)) is not None:
return IssueSummaryResult(outcome=blocked, summary=stored)
- if (held := _claim_generation(team_id, issue_id)) is not None:
+ claim = f"{LOCK_CLAIM_PREFIX}{uuid4().hex}"
+ if (held := _claim_generation(team_id, issue_id, claim)) is not None:
return IssueSummaryResult(outcome=held, summary=stored)
+ try:
+ result = _generate_claimed(team_id, issue_id)
+ except Exception:
+ _settle_claim(team_id, issue_id, claim, IssueSummaryOutcome.UNEXPECTED_ERROR)
+ raise
+ _settle_claim(team_id, issue_id, claim, result.outcome)
+ return result
+
+
+def _generate_claimed(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:
# Another worker can finish between the first read and the claim.
stored = get_issue_summary(team_id, issue_id)
if not regeneration_due(stored, timezone.now()):
@@ -239,7 +360,6 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:
# The input queries run before `consume`, so an unchanged issue spends no token. This read keeps a team
# with no tokens left from running them for every issue it opens. `consume` still makes the decision.
if _budget_exhausted(budget_key):
- _hold_generation(team_id, issue_id, IssueSummaryOutcome.RATE_LIMITED)
return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)
team = Team.objects.get(id=team_id)
@@ -250,8 +370,6 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:
logger.warning("error_tracking_issue_summary_inputs_failed", team_id=team_id, issue_id=str(issue_id))
return IssueSummaryResult(outcome=IssueSummaryOutcome.INPUTS_ERROR, summary=stored)
if isinstance(inputs, IssueSummaryOutcome):
- if inputs == IssueSummaryOutcome.NO_EXCEPTION:
- _hold_generation(team_id, issue_id, inputs)
return IssueSummaryResult(outcome=inputs, summary=stored)
if stored is not None and stored.input_hash == inputs.input_hash:
stored.checked_at = timezone.now()
@@ -261,7 +379,6 @@ def _get_or_generate(team_id: int, issue_id: UUID | str) -> IssueSummaryResult:
decision = consume(budget_key, ISSUE_SUMMARY_BUDGET)
match decision:
case BucketDecision(allowed=False):
- _hold_generation(team_id, issue_id, IssueSummaryOutcome.RATE_LIMITED)
return IssueSummaryResult(outcome=IssueSummaryOutcome.RATE_LIMITED, summary=stored)
case BucketUnavailable(error=bucket_error):
# Fail open: the per-issue lock and the regenerate interval still bound the spend.
@@ -313,28 +430,39 @@ def _lock_key(team_id: int, issue_id: UUID | str) -> str:
return f"{ISSUE_SUMMARY_LOCK_KEY_PREFIX}:{team_id}:{issue_id}"
-def _claim_generation(team_id: int, issue_id: UUID | str) -> IssueSummaryOutcome | None:
- """None when this worker now holds the issue, else the outcome that holds it."""
+def _claim_generation(team_id: int, issue_id: UUID | str, claim: str) -> IssueSummaryOutcome | None:
+ """None when this worker now holds the issue under `claim`, else the outcome that holds it."""
try:
- client = get_client()
- key = _lock_key(team_id, issue_id)
- if client.set(key, LOCK_HELD_FOR_GENERATION, nx=True, ex=ISSUE_SUMMARY_LOCK_TTL_SECONDS):
- return None
- held = client.get(key)
+ held = get_client().register_script(CLAIM_LUA)(
+ keys=[_lock_key(team_id, issue_id)],
+ args=[LOCK_QUEUED, claim, ISSUE_SUMMARY_LOCK_TTL_SECONDS],
+ )
except RedisError:
# Fail open, as the budget does: a Redis outage can let two concurrent reads both pay.
logger.warning("error_tracking_issue_summary_lock_unavailable", team_id=team_id, exc_info=True)
return None
+ return _lock_outcome(held) if held is not None else None
+
+
+def _lock_outcome(held: bytes | str | None) -> IssueSummaryOutcome:
+ """A live claim or a queued task means a summary may come. Every other value is the outcome that stopped generation."""
value = held.decode() if isinstance(held, bytes) else held
- if value in (IssueSummaryOutcome.NO_EXCEPTION, IssueSummaryOutcome.RATE_LIMITED):
- return IssueSummaryOutcome(value)
- return IssueSummaryOutcome.IN_PROGRESS
+ if value is None or value in (LEGACY_LOCK_CLAIM, LOCK_QUEUED) or value.startswith(LOCK_CLAIM_PREFIX):
+ return IssueSummaryOutcome.IN_PROGRESS
+ return IssueSummaryOutcome(value)
+
+def _settle_claim(team_id: int, issue_id: UUID | str, claim: str, outcome: IssueSummaryOutcome) -> None:
+ """Replace this worker's claim with `outcome` when no summary came of it, so reads stop waiting for one.
-def _hold_generation(team_id: int, issue_id: UUID | str, outcome: IssueSummaryOutcome) -> None:
- """Keep the issue claimed past the lock TTL, so reads return `outcome` without running the queries again."""
+ A hold outcome extends the lock. Any other failure keeps the lock TTL, so the retry spacing does not change.
+ Nothing is written when the key no longer holds `claim`: the lock expired, or another read took the issue.
+ """
+ if outcome in SETTLED_OUTCOMES:
+ return
+ ttl = str(ISSUE_SUMMARY_HOLD_SECONDS) if outcome in HOLD_OUTCOMES else ""
try:
- get_client().set(_lock_key(team_id, issue_id), outcome.value, ex=ISSUE_SUMMARY_HOLD_SECONDS)
+ get_client().register_script(SETTLE_LUA)(keys=[_lock_key(team_id, issue_id)], args=[claim, outcome.value, ttl])
except RedisError:
logger.warning("error_tracking_issue_summary_hold_unavailable", team_id=team_id, exc_info=True)
@@ -350,6 +478,13 @@ def compute_input_hash(*, fingerprints: list[str], latest_release: str | None, o
return hashlib.sha256(json.dumps(payload, sort_keys=True).encode()).hexdigest()
+# The only event properties the prompt builder sees, and the ones `summary_sources_readable` checks.
+# `_exception_event_properties` drops every other key, so a property left out is missing from the prompt, not leaked.
+ISSUE_SUMMARY_SOURCE_PROPERTIES = frozenset(
+ {"$exception_list", "$exception_level", "$lib", "$current_url", "$exception_release"}
+)
+
+
def build_issue_summary_inputs(team: Team, issue_id: UUID | str) -> IssueSummaryInputs | IssueSummaryOutcome:
"""Build the prompt and the input hash, or say why the issue cannot be summarised."""
issue = ErrorTrackingIssue.objects.with_first_seen().filter(team_id=team.id, id=issue_id).first()
@@ -496,7 +631,8 @@ def _exception_event_properties(
)
if not response.results:
return {}
- return as_record(parse_jsonish(response.results[0][0])) or {}
+ properties = as_record(parse_jsonish(response.results[0][0])) or {}
+ return {key: value for key, value in properties.items() if key in ISSUE_SUMMARY_SOURCE_PROPERTIES}
def _aggregated_issue(team: Team, issue_id: UUID | str) -> dict[str, object]:
diff --git a/products/error_tracking/backend/logic/issue_summary_config.py b/products/error_tracking/backend/logic/issue_summary_config.py
index 8384cd972727..b69c01c074b6 100644
--- a/products/error_tracking/backend/logic/issue_summary_config.py
+++ b/products/error_tracking/backend/logic/issue_summary_config.py
@@ -25,6 +25,11 @@ ISSUE_SUMMARY_MIN_REGENERATE_INTERVAL = timedelta(hours=24)
# Holds one generation per issue at a time. It is never released, so it also spaces out retries after a failure.
ISSUE_SUMMARY_LOCK_TTL_SECONDS = 300
+# Marks a queued task until a worker claims it, so page polls queue at most one task per issue in this window.
+# A lost task keeps reads on "generating" for this long, so it stays short.
+ISSUE_SUMMARY_QUEUED_TTL_SECONDS = 60
+# Bounds the lock calls and the publish when a read queues a task, so a slow Redis cannot hold the request.
+ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS = 1.0
# An issue with no recent exception event, or a team out of budget, is not tried again for this long.
# Each try runs the ClickHouse queries before it can find either outcome.
ISSUE_SUMMARY_HOLD_SECONDS = 3600
diff --git a/products/error_tracking/backend/tests/test_issue_summaries.py b/products/error_tracking/backend/tests/test_issue_summaries.py
index eae757f021b6..ab15ae21d127 100644
--- a/products/error_tracking/backend/tests/test_issue_summaries.py
+++ b/products/error_tracking/backend/tests/test_issue_summaries.py
@@ -1,4 +1,6 @@
import json
+import signal
+import socket
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -6,14 +8,19 @@ import time_machine
from posthog.test.base import BaseTest, ClickhouseTestMixin, _create_event, flush_persons_and_events
from unittest.mock import MagicMock, patch
+from django.db import OperationalError
from django.test import SimpleTestCase
from parameterized import parameterized
from redis.exceptions import RedisError
+from posthog.constants import AvailableFeature
+from posthog.models import PropertyDefinition, User
from posthog.redis import TEST_clear_clients
-from posthog.token_bucket import BucketUnavailable, Budget, TEST_reset_scripts, consume
+from posthog.token_bucket import BucketUnavailable, Budget, TEST_reset_scripts, consume, refund
+from products.access_control.backend.facade.api import upsert_property_access_control
+from products.access_control.backend.facade.contracts import PropertyAccessLevel, UpsertPropertyAccessControlInput
from products.error_tracking.backend.facade import summaries as summaries_facade
from products.error_tracking.backend.facade.contracts import IssueSummaryCategory
from products.error_tracking.backend.logic import issue_summaries, team_flags
@@ -91,6 +98,11 @@ FAILURE_CASES: list[tuple[str, Any, IssueSummaryOutcome]] = [
IssueSummaryOutcome.INPUTS_ERROR,
),
("cap_denied", lambda test: test._drain_budget(), IssueSummaryOutcome.RATE_LIMITED),
+ (
+ "gateway_unconfigured",
+ lambda test: test.enterContext(patch.object(issue_summaries, "resolve_ai_gateway_config", return_value=None)),
+ IssueSummaryOutcome.GATEWAY_UNCONFIGURED,
+ ),
(
"llm_raises",
lambda test: test.completions.configure_mock(side_effect=TimeoutError("slow")),
@@ -137,10 +149,12 @@ class TestIssueSummaryGeneration(BaseTest):
def llm_calls(self) -> int:
return self.completions.call_count
+ @property
+ def _lock_key(self) -> str:
+ return issue_summaries._lock_key(self.team.id, self.issue.id)
+
def _release_lock(self) -> None:
- issue_summaries.get_client().delete(
- f"{issue_summaries.ISSUE_SUMMARY_LOCK_KEY_PREFIX}:{self.team.id}:{self.issue.id}"
- )
+ issue_summaries.get_client().delete(self._lock_key)
@parameterized.expand(
[
@@ -157,7 +171,7 @@ class TestIssueSummaryGeneration(BaseTest):
result = get_or_generate_issue_summary(self.team.id, self.issue.id)
assert result.outcome == IssueSummaryOutcome.GENERATED
- summary = summaries_facade.get_issue_summary(self.team.id, self.issue.id)
+ summary = summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=self.user)
assert summary is not None
assert summary.what_happened == SUMMARY_REPLY["what_happened"]
assert summary.category == expected_category
@@ -179,10 +193,17 @@ class TestIssueSummaryGeneration(BaseTest):
assert result.outcome == IssueSummaryOutcome.GENERATED
+ @property
+ def _budget_key(self) -> str:
+ return f"{issue_summaries.ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{self.team.id}"
+
def _drain_budget(self) -> None:
tiny = Budget(burst=1, per_hour=1)
self.enterContext(patch.object(issue_summaries, "ISSUE_SUMMARY_BUDGET", tiny))
- consume(f"{issue_summaries.ISSUE_SUMMARY_BUDGET_KEY_PREFIX}:{self.team.id}", tiny)
+ consume(self._budget_key, tiny)
+
+ def _refill_budget(self) -> None:
+ refund(self._budget_key, issue_summaries.ISSUE_SUMMARY_BUDGET)
def _disapprove_ai(self) -> None:
self.organization.is_ai_data_processing_approved = False
@@ -198,13 +219,6 @@ class TestIssueSummaryGeneration(BaseTest):
IssueSummaryOutcome.DISABLED,
),
("ai_not_approved", lambda test: test._disapprove_ai(), IssueSummaryOutcome.AI_NOT_APPROVED),
- (
- "gateway_unconfigured",
- lambda test: test.enterContext(
- patch.object(issue_summaries, "resolve_ai_gateway_config", return_value=None)
- ),
- IssueSummaryOutcome.GATEWAY_UNCONFIGURED,
- ),
*FAILURE_CASES,
]
)
@@ -251,6 +265,7 @@ class TestIssueSummaryGeneration(BaseTest):
arrange(self)
with time_machine.travel(NOW, tick=False):
get_or_generate_issue_summary(self.team.id, self.issue.id)
+ self._refill_budget()
with time_machine.travel(NOW + timedelta(minutes=10), tick=False):
result = get_or_generate_issue_summary(self.team.id, self.issue.id)
@@ -289,16 +304,19 @@ class TestIssueSummaryGeneration(BaseTest):
with (
time_machine.travel(NOW + later, tick=False),
- patch.object(issue_summaries.generate_error_tracking_issue_summary, "delay") as delay,
+ patch.object(issue_summaries.generate_error_tracking_issue_summary, "apply_async") as apply_async,
):
result = get_or_generate_issue_summary(self.team.id, self.issue.id)
- assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id) is False
- delay.assert_not_called()
+ assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user) is False
+ apply_async.assert_not_called()
assert result.outcome == expected
assert result.summary is not None
assert self.llm_calls == (2 if expected == IssueSummaryOutcome.GENERATED else 1)
assert result.summary.input_hash == (new_hash if expected == IssueSummaryOutcome.GENERATED else "hash-1")
+ shown = summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=self.user)
+ assert shown is not None
+ assert (shown.generated_at, shown.checked_at) == (result.summary.generated_at, result.summary.checked_at)
def test_merge_keeps_target_summary_and_deletes_source_summary(self) -> None:
source = ErrorTrackingIssue.objects.create(team=self.team, name="TypeError")
@@ -313,19 +331,202 @@ class TestIssueSummaryGeneration(BaseTest):
)
assert stored_issue_ids == {self.issue.id}
+ def _start(self) -> tuple[bool, bool]:
+ with patch.object(issue_summaries.generate_error_tracking_issue_summary, "apply_async") as apply_async:
+ started = summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user)
+ return started, apply_async.called
+
+ @parameterized.expand(
+ [
+ ("due", True, None, True, True),
+ ("flag_off", False, None, False, False),
+ ("held_no_exception", True, IssueSummaryOutcome.NO_EXCEPTION.value, False, False),
+ ("held_rate_limited", True, IssueSummaryOutcome.RATE_LIMITED.value, False, False),
+ ("held_after_failure", True, IssueSummaryOutcome.MODEL_ERROR.value, False, False),
+ ("claimed_by_another_worker", True, f"{issue_summaries.LOCK_CLAIM_PREFIX}other", True, False),
+ ("claimed_by_an_older_worker", True, issue_summaries.LEGACY_LOCK_CLAIM, True, False),
+ ]
+ )
+ def test_start_generation_queues_only_when_it_can_generate(
+ self, _name: str, flag: bool, lock_value: str | None, started: bool, queued: bool
+ ) -> None:
+ if lock_value is not None:
+ issue_summaries.get_client().set(self._lock_key, lock_value, ex=60)
+
+ with patch.object(team_flags, "feature_enabled_or_false", return_value=flag):
+ assert self._start() == (started, queued)
+
+ def test_start_generation_queues_when_redis_is_down(self) -> None:
+ with patch.object(issue_summaries, "get_client", side_effect=RedisError("down")):
+ assert self._start() == (True, True)
+
+ def test_start_queues_one_task_until_a_worker_claims_it(self) -> None:
+ assert self._start() == (True, True)
+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= issue_summaries.ISSUE_SUMMARY_QUEUED_TTL_SECONDS
+ assert self._start() == (True, False)
+
+ result = get_or_generate_issue_summary(self.team.id, self.issue.id)
+
+ assert result.outcome == IssueSummaryOutcome.GENERATED
+
+ def test_start_returns_false_when_the_broker_is_down(self) -> None:
+ capture = self.enterContext(patch.object(issue_summaries, "capture_exception"))
+ with patch.object(
+ issue_summaries.generate_error_tracking_issue_summary, "apply_async", side_effect=OSError("broker down")
+ ):
+ assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user) is False
+
+ capture.assert_called_once()
+ assert self._start() == (True, True)
+
@parameterized.expand(
[
- ("due", True, True),
- ("flag_off", False, False),
+ ("llm_raises", lambda test: test.completions.configure_mock(side_effect=TimeoutError("slow"))),
+ (
+ "issue_missing",
+ lambda test: test.build_inputs.configure_mock(
+ side_effect=None, return_value=IssueSummaryOutcome.ISSUE_MISSING
+ ),
+ ),
]
)
- def test_start_generation_queues_only_when_it_can_generate(self, _name: str, flag: bool, queued: bool) -> None:
+ def test_failure_stops_start_until_the_lock_expires(self, _name: str, arrange: Any) -> None:
+ arrange(self)
+ get_or_generate_issue_summary(self.team.id, self.issue.id)
+
+ assert self._start() == (False, False)
+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= issue_summaries.ISSUE_SUMMARY_LOCK_TTL_SECONDS
+
+ self._release_lock()
+ assert self._start() == (True, True)
+
+ @parameterized.expand(
+ [
+ ("failure", "completions", TimeoutError("slow"), IssueSummaryOutcome.MODEL_ERROR),
+ ("hold", "build_inputs", IssueSummaryOutcome.NO_EXCEPTION, IssueSummaryOutcome.NO_EXCEPTION),
+ ]
+ )
+ def test_settle_does_not_overwrite_a_lock_this_worker_no_longer_holds(
+ self, _name: str, mock_name: str, result_after: Any, expected: IssueSummaryOutcome
+ ) -> None:
+ def lose_the_lock(*_args: Any, **_kwargs: Any) -> Any:
+ issue_summaries.get_client().set(self._lock_key, issue_summaries.LOCK_QUEUED, ex=60)
+ if isinstance(result_after, Exception):
+ raise result_after
+ return result_after
+
+ getattr(self, mock_name).side_effect = lose_the_lock
+
+ result = get_or_generate_issue_summary(self.team.id, self.issue.id)
+
+ assert result.outcome == expected
+ assert issue_summaries.get_client().get(self._lock_key) == issue_summaries.LOCK_QUEUED.encode()
+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= 60
+
+ def test_unexpected_error_stops_start_and_still_raises(self) -> None:
with (
- patch.object(team_flags, "feature_enabled_or_false", return_value=flag),
- patch.object(issue_summaries.generate_error_tracking_issue_summary, "delay") as delay,
+ patch.object(issue_summaries.Team.objects, "get", side_effect=OperationalError("db down")),
+ self.assertRaises(OperationalError),
):
- assert summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id) == queued
- assert delay.called == queued
+ get_or_generate_issue_summary(self.team.id, self.issue.id)
+
+ assert self._start() == (False, False)
+ assert 0 < issue_summaries.get_client().ttl(self._lock_key) <= issue_summaries.ISSUE_SUMMARY_LOCK_TTL_SECONDS
+
+ @parameterized.expand(
+ [
+ ("get", lambda test: summaries_facade.get_issue_summary(test.team.id, test.issue.id, user=test.user)),
+ (
+ "wait",
+ lambda test: summaries_facade.wait_for_issue_summary(
+ test.team.id, test.issue.id, user=test.user, checked_after=None, timeout_seconds=0
+ ),
+ ),
+ (
+ "get_or_generate",
+ lambda test: summaries_facade.get_or_generate_issue_summary(
+ test.team.id, test.issue.id, user=test.user
+ ),
+ ),
+ ]
+ )
+ def test_flag_off_or_restricted_source_hides_stored_summary(self, _name: str, read: Any) -> None:
+ get_or_generate_issue_summary(self.team.id, self.issue.id)
+ assert read(self) is not None
+
+ with patch.object(team_flags, "feature_enabled_or_false", return_value=False):
+ assert read(self) is None
+
+ self._restrict_event_property("$current_url")
+ assert read(self) is None
+
+ def _restrict_event_property(self, name: str) -> None:
+ self.organization.available_product_features = [
+ {"name": AvailableFeature.PROPERTY_ACCESS_CONTROL, "key": AvailableFeature.PROPERTY_ACCESS_CONTROL}
+ ]
+ self.organization.save()
+ definition = PropertyDefinition.objects.create(team=self.team, name=name, type=PropertyDefinition.Type.EVENT)
+ upsert_property_access_control(
+ team_id=self.team.id,
+ created_by_id=self.user.id,
+ input=UpsertPropertyAccessControlInput(
+ property_definition_id=str(definition.id),
+ access_level=PropertyAccessLevel.NONE,
+ organization_member_id=self.organization_membership.id,
+ ),
+ )
+
+ def test_restricted_member_cannot_start_and_other_members_still_read(self) -> None:
+ other = User.objects.create_and_join(self.organization, "[email protected]", "password")
+ self._restrict_event_property("$exception_list")
+
+ assert self._start() == (False, False)
+ assert summaries_facade.get_or_generate_issue_summary(self.team.id, self.issue.id, user=self.user) is None
+ assert self.llm_calls == 0
+
+ get_or_generate_issue_summary(self.team.id, self.issue.id)
+ assert summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=other) is not None
+ outsider = User.objects.create(email="[email protected]")
+ assert summaries_facade.get_issue_summary(self.team.id, self.issue.id, user=outsider) is None
+
+ def test_start_gives_up_quickly_when_the_broker_does_not_answer(self) -> None:
+ # Not an Exception, because `_queue_generation` catches Exception and would turn a hung publish into a pass.
+ class StartBlocked(BaseException):
+ pass
+
+ def reset_producer_pool() -> None:
+ app._pool = None
+ app.amqp._producer_pool = None
+
+ self.enterContext(patch.object(issue_summaries, "ISSUE_SUMMARY_PUBLISH_TIMEOUT_SECONDS", 0.05))
+ silent_broker = socket.socket()
+ silent_broker.bind(("127.0.0.1", 0))
+ silent_broker.listen()
+ self.addCleanup(silent_broker.close)
+ app = issue_summaries.generate_error_tracking_issue_summary.app
+ silent_url = f"redis://127.0.0.1:{silent_broker.getsockname()[1]}/0"
+ # Settings name the broker with the old CELERY_ key, which wins over `broker_url`.
+ saved = {key: app.conf[key] for key in ("task_always_eager", "CELERY_BROKER_URL")}
+ # The producer pool keeps the broker it first connected to, so drop it on both sides of the swap.
+ self.addCleanup(reset_producer_pool)
+ self.addCleanup(app.conf.update, saved)
+ app.conf.update(task_always_eager=False, CELERY_BROKER_URL=silent_url)
+ reset_producer_pool()
+ assert app.connection_for_write().as_uri() == silent_url
+ self.enterContext(patch.object(issue_summaries, "capture_exception"))
+
+ def hung(*_args: Any) -> None:
+ raise StartBlocked
+
+ previous = signal.signal(signal.SIGALRM, hung)
+ self.addCleanup(signal.signal, signal.SIGALRM, previous)
+ signal.alarm(3)
+ try:
+ started = summaries_facade.start_issue_summary_generation(self.team.id, self.issue.id, user=self.user)
+ finally:
+ signal.alarm(0)
+
+ assert started is False
@parameterized.expand(
[
@@ -341,7 +542,7 @@ class TestIssueSummaryGeneration(BaseTest):
get_or_generate_issue_summary(self.team.id, self.issue.id)
summary = summaries_facade.wait_for_issue_summary(
- self.team.id, self.issue.id, checked_after=checked_after, timeout_seconds=0
+ self.team.id, self.issue.id, user=self.user, checked_after=checked_after, timeout_seconds=0
)
assert (summary is not None) == found