Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(error-tracking): stop concurrent symbol set uploads from deadlocking (#123545)

Vasco de Krijger4f21b1fdfb49
diff --git a/products/error_tracking/backend/logic/symbol_sets.py b/products/error_tracking/backend/logic/symbol_sets.py
index b7fafd4e9cc6..e3227702eddf 100644
--- a/products/error_tracking/backend/logic/symbol_sets.py
+++ b/products/error_tracking/backend/logic/symbol_sets.py
@@ -12,10 +12,11 @@ from collections import Counter
from typing import Any
from django.conf import settings
-from django.db import transaction
+from django.db import OperationalError, transaction
from django.db.models import Q
from django.utils import timezone
+import psycopg
import structlog
import posthoganalytics
from rest_framework.exceptions import ValidationError
@@ -40,6 +41,8 @@ PRESIGNED_MULTIPLE_UPLOAD_TIMEOUT = 60 * 5
# `last_used` writes over the same window for the same reason.
LAST_USED_UPLOAD_REFRESH_INTERVAL = datetime.timedelta(hours=12)
+DEADLOCK_RETRY_ATTEMPTS = 3
+
# The ID tiebreak keeps LIMIT/OFFSET paging stable when rows share a timestamp.
# The reference is unique within a team, so it does not need a tiebreak.
SYMBOL_SET_ORDERINGS: dict[str, tuple[str, ...]] = {
@@ -192,11 +195,18 @@ def refresh_last_used(team: Team, chunk_ids: list[str]) -> None:
symbolicated until a later deploy recreates the row.
"""
now = timezone.now()
- ErrorTrackingSymbolSet.objects.filter(
- Q(last_used__isnull=True) | Q(last_used__lt=now - LAST_USED_UPLOAD_REFRESH_INTERVAL),
- team=team,
- ref__in=chunk_ids,
- ).update(last_used=now)
+ with transaction.atomic():
+ stale_ids = list(
+ ErrorTrackingSymbolSet.objects.filter(
+ Q(last_used__isnull=True) | Q(last_used__lt=now - LAST_USED_UPLOAD_REFRESH_INTERVAL),
+ team=team,
+ ref__in=chunk_ids,
+ )
+ .order_by("ref")
+ .select_for_update(no_key=True)
+ .values_list("id", flat=True)
+ )
+ ErrorTrackingSymbolSet.objects.filter(team=team, id__in=stale_ids).update(last_used=now)
def _validate_uploads(new_symbol_sets: list[SymbolSetUpload], team: Team) -> None:
@@ -336,21 +346,30 @@ def bulk_check_symbol_sets(
return chunk_ids_to_upload
[email protected](capture_exceptions=False)
-def bulk_create_symbol_sets(
+def _lock_symbol_sets(team: Team, refs: list[str]) -> list[ErrorTrackingSymbolSet]:
+ return list(
+ ErrorTrackingSymbolSet.objects.filter(team=team, ref__in=refs).order_by("ref").select_for_update(no_key=True)
+ )
+
+
+def _create_or_reuse_symbol_sets(
new_symbol_sets: list[SymbolSetUpload],
team: Team,
- force: bool = False,
- skip_on_conflict: bool = False,
+ force: bool,
+ skip_on_conflict: bool,
) -> dict[str, dict[str, Any]]:
- _validate_uploads(new_symbol_sets, team)
- chunk_ids = [x.chunk_id for x in new_symbol_sets]
+ chunk_ids = sorted(x.chunk_id for x in new_symbol_sets)
id_url_map: dict[str, dict[str, Any]] = {}
new_symbol_set_map = {x.chunk_id: x for x in new_symbol_sets}
def reissue_upload(existing: ErrorTrackingSymbolSet) -> None:
- storage_ptr = generate_symbol_set_file_key()
+ # Pending uploads share an object key so a retry cannot invalidate an in-flight upload URL.
+ storage_ptr = (
+ existing.storage_ptr
+ if existing.content_hash is None and existing.storage_ptr
+ else generate_symbol_set_file_key()
+ )
id_url_map[existing.ref] = {
**generate_symbol_set_upload_presigned_urls(storage_ptr, new_symbol_set_map[existing.ref].content_length),
"symbol_set_id": str(existing.id),
@@ -360,9 +379,9 @@ def bulk_create_symbol_sets(
existing.content_hash = None
with transaction.atomic():
- existing_symbol_sets = list(ErrorTrackingSymbolSet.objects.filter(team=team, ref__in=chunk_ids))
- existing_symbol_set_refs = [s.ref for s in existing_symbol_sets]
- missing_sets = list(set(chunk_ids) - set(existing_symbol_set_refs))
+ existing_symbol_sets = _lock_symbol_sets(team, chunk_ids)
+ existing_symbol_set_refs = {s.ref for s in existing_symbol_sets}
+ missing_sets = [chunk_id for chunk_id in chunk_ids if chunk_id not in existing_symbol_set_refs]
symbol_sets_to_be_created = []
for chunk_id in missing_sets:
@@ -382,10 +401,15 @@ def bulk_create_symbol_sets(
symbol_sets_to_be_created.append(to_create)
# create missing symbol sets
- created_symbol_sets = ErrorTrackingSymbolSet.objects.bulk_create(symbol_sets_to_be_created)
+ ErrorTrackingSymbolSet.objects.bulk_create(symbol_sets_to_be_created, ignore_conflicts=True)
+ storage_ptrs_we_created = {s.ref: s.storage_ptr for s in symbol_sets_to_be_created}
- for symbol_set in created_symbol_sets:
- id_url_map[symbol_set.ref]["symbol_set_id"] = str(symbol_set.pk)
+ for symbol_set in _lock_symbol_sets(team, missing_sets):
+ if symbol_set.storage_ptr == storage_ptrs_we_created[symbol_set.ref]:
+ id_url_map[symbol_set.ref]["symbol_set_id"] = str(symbol_set.pk)
+ else:
+ del id_url_map[symbol_set.ref]
+ existing_symbol_sets.append(symbol_set)
# update existing symbol sets
to_update = []
@@ -413,6 +437,24 @@ def bulk_create_symbol_sets(
return id_url_map
[email protected](capture_exceptions=False)
+def bulk_create_symbol_sets(
+ new_symbol_sets: list[SymbolSetUpload],
+ team: Team,
+ force: bool = False,
+ skip_on_conflict: bool = False,
+) -> dict[str, dict[str, Any]]:
+ _validate_uploads(new_symbol_sets, team)
+ for attempt in range(1, DEADLOCK_RETRY_ATTEMPTS):
+ try:
+ return _create_or_reuse_symbol_sets(new_symbol_sets, team, force, skip_on_conflict)
+ except OperationalError as e:
+ if not isinstance(e.__cause__, psycopg.errors.DeadlockDetected):
+ raise
+ logger.warning("symbol_set_upload_deadlock_retry", team_id=team.id, attempt=attempt)
+ return _create_or_reuse_symbol_sets(new_symbol_sets, team, force, skip_on_conflict)
+
+
def list_symbol_sets(
team_id: int,
*,
@@ -629,7 +671,9 @@ def bulk_finish_upload(
content_hash = content_hashes[str(symbol_set.id)]
symbol_set.content_hash = content_hash
- ErrorTrackingSymbolSet.objects.bulk_update(symbol_sets, ["content_hash"])
+ with transaction.atomic():
+ _lock_symbol_sets(team, [symbol_set.ref for symbol_set in symbol_sets])
+ ErrorTrackingSymbolSet.objects.bulk_update(symbol_sets, ["content_hash"])
except Exception as e:
posthoganalytics.capture(
"error_tracking_symbol_set_uploaded",
diff --git a/products/error_tracking/backend/tests/api/test_error_tracking_api.py b/products/error_tracking/backend/tests/api/test_error_tracking_api.py
index 6cc84c7315af..a6566790dd04 100644
--- a/products/error_tracking/backend/tests/api/test_error_tracking_api.py
+++ b/products/error_tracking/backend/tests/api/test_error_tracking_api.py
@@ -2,17 +2,19 @@ import os
import json
from collections.abc import Callable
from datetime import UTC, datetime, timedelta
+from typing import Any
from urllib.parse import unquote
import time_machine
from posthog.test.base import APIBaseTest
from unittest.mock import ANY, Mock, patch
-from django.db import connection
+from django.db import OperationalError, connection
from django.test import SimpleTestCase
from django.test.utils import CaptureQueriesContext
from django.utils import timezone
+import psycopg
from boto3 import resource
from botocore.config import Config
from parameterized import parameterized
@@ -36,6 +38,7 @@ from products.error_tracking.backend.facade import (
contracts,
summaries as summaries_facade,
)
+from products.error_tracking.backend.logic import symbol_sets as symbol_sets_logic
from products.error_tracking.backend.models import (
ErrorTrackingAlert,
ErrorTrackingAlertThread,
@@ -1693,6 +1696,107 @@ class TestErrorTracking(APIBaseTest):
assert "fallback_presigned_url" not in symbol_set_upload_response
assert symbol_set.last_used is not None
+ @parameterized.expand(
+ [
+ ("pending", None, True),
+ ("placeholder", None, True),
+ ("uploaded", None, False),
+ ("content_mismatch", "content_hash_mismatch", False),
+ ("release_mismatch", "release_id_mismatch", False),
+ ]
+ )
+ def test_bulk_start_upload_reuses_symbol_set_created_by_concurrent_upload(
+ self, existing_state: str, expected_code: str | None, expect_upload: bool
+ ) -> None:
+ chunk_id = str(uuid7())
+ new_chunk_id = str(uuid7())
+ release = ErrorTrackingRelease.objects.create(
+ team=self.team, hash_id="upload-release", version="1.0.0", project="test"
+ )
+ other_release = ErrorTrackingRelease.objects.create(
+ team=self.team, hash_id="other-upload-release", version="2.0.0", project="test"
+ )
+ inserted = False
+ winner: ErrorTrackingSymbolSet | None = None
+
+ def insert_before_bulk_create(
+ execute: Callable[..., object], sql: str, params: object, many: bool, context: object
+ ) -> object:
+ nonlocal inserted, winner
+ if not inserted and sql.startswith('INSERT INTO "posthog_errortrackingsymbolset"'):
+ inserted = True
+ winner = ErrorTrackingSymbolSet.objects.create(
+ team=self.team,
+ ref=chunk_id,
+ storage_ptr=None if existing_state == "placeholder" else "symbolsets/concurrent-upload",
+ content_hash=("hash" if existing_state == "uploaded" else "other_hash" if expected_code else None),
+ release=other_release if existing_state == "release_mismatch" else None,
+ )
+ return execute(sql, params, many, context)
+
+ with connection.execute_wrapper(insert_before_bulk_create):
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/error_tracking/symbol_sets/bulk_start_upload",
+ data={
+ "symbol_sets": [
+ {"chunk_id": chunk_id, "content_hash": "hash", "release_id": str(release.id)},
+ {"chunk_id": new_chunk_id, "content_hash": "new_hash"},
+ ]
+ },
+ format="json",
+ )
+
+ assert winner is not None
+ if expected_code:
+ assert response.status_code == status.HTTP_400_BAD_REQUEST
+ assert response.json()["code"] == expected_code
+ assert not ErrorTrackingSymbolSet.objects.filter(team=self.team, ref=new_chunk_id).exists()
+ return
+
+ assert response.status_code == status.HTTP_201_CREATED
+ id_map = response.json()["id_map"]
+ symbol_set = ErrorTrackingSymbolSet.objects.get(team=self.team, ref=chunk_id)
+ assert symbol_set.id == winner.id
+ assert symbol_set.release_id == release.id
+ assert (chunk_id in id_map) == expect_upload
+ if expect_upload:
+ assert id_map[chunk_id]["symbol_set_id"] == str(winner.id)
+ assert id_map[chunk_id]["presigned_url"]["fields"]["key"] == symbol_set.storage_ptr
+ if existing_state == "pending":
+ assert symbol_set.storage_ptr == winner.storage_ptr
+ else:
+ assert symbol_set.content_hash == "hash"
+ assert symbol_set.storage_ptr == winner.storage_ptr
+ new_symbol_set = ErrorTrackingSymbolSet.objects.get(team=self.team, ref=new_chunk_id)
+ assert id_map[new_chunk_id]["symbol_set_id"] == str(new_symbol_set.id)
+ assert id_map[new_chunk_id]["presigned_url"]["fields"]["key"] == new_symbol_set.storage_ptr
+
+ def test_bulk_start_upload_retries_after_deadlock(self) -> None:
+ chunk_id = str(uuid7())
+ deadlock = OperationalError("deadlock detected")
+ deadlock.__cause__ = psycopg.errors.DeadlockDetected("deadlock detected")
+ create_or_reuse = symbol_sets_logic._create_or_reuse_symbol_sets
+ calls = 0
+
+ def deadlock_on_first_attempt(*args: Any) -> dict[str, dict[str, Any]]:
+ nonlocal calls
+ calls += 1
+ if calls == 1:
+ raise deadlock
+ return create_or_reuse(*args)
+
+ with patch.object(symbol_sets_logic, "_create_or_reuse_symbol_sets", side_effect=deadlock_on_first_attempt):
+ response = self.client.post(
+ f"/api/environments/{self.team.id}/error_tracking/symbol_sets/bulk_start_upload",
+ data={"chunk_ids": [chunk_id]},
+ )
+
+ assert response.status_code == status.HTTP_201_CREATED
+ assert calls == 2
+ assert response.json()["id_map"][chunk_id]["symbol_set_id"] == str(
+ ErrorTrackingSymbolSet.objects.get(team=self.team, ref=chunk_id).id
+ )
+
def test_bulk_start_upload_includes_fallback_presigned_url_when_accelerated(self) -> None:
chunk_id = str(uuid7())
with (
@@ -2060,8 +2164,9 @@ class TestErrorTracking(APIBaseTest):
symbol_set.refresh_from_db()
assert symbol_set.release_id == release.id
- assert symbol_set.storage_ptr != initial_storage_ptr
+ assert symbol_set.storage_ptr == initial_storage_ptr
assert id_map[str(chunk_id)]["symbol_set_id"] == str(symbol_set.id)
+ assert id_map[str(chunk_id)]["presigned_url"]["fields"]["key"] == initial_storage_ptr
def test_bulk_start_upload_updates_release_for_uploaded_symbol_set(self) -> None:
chunk_id = str(uuid7())
@@ -2128,8 +2233,9 @@ class TestErrorTracking(APIBaseTest):
id_map = second_response.json()["id_map"]
symbol_set.refresh_from_db()
- assert symbol_set.storage_ptr != initial_storage_ptr
+ assert symbol_set.storage_ptr == initial_storage_ptr
assert id_map[str(chunk_id)]["symbol_set_id"] == str(symbol_set.id)
+ assert id_map[str(chunk_id)]["presigned_url"]["fields"]["key"] == initial_storage_ptr
def test_bulk_start_upload_rejects_release_change(self) -> None:
chunk_id = str(uuid7())