:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(error-tracking): stop concurrent symbol set uploads from deadlocking (#123545)
Vasco de Krijger4f21b1fdfb49
diff --git a/products/error_tracking/backend/logic/symbol_sets.py b/products/error_tracking/backend/logic/symbol_sets.pyindex b7fafd4e9cc6..e3227702eddf 100644--- a/products/error_tracking/backend/logic/symbol_sets.py+++ b/products/error_tracking/backend/logic/symbol_sets.py@@ -12,10 +12,11 @@ from collections import Counterfrom typing import Anyfrom django.conf import settings-from django.db import transaction+from django.db import OperationalError, transactionfrom django.db.models import Qfrom django.utils import timezone+import psycopgimport structlogimport posthoganalyticsfrom rest_framework.exceptions import ValidationError@@ -40,6 +41,8 @@ PRESIGNED_MULTIPLE_UPLOAD_TIMEOUT = 60 * 5# `last_used` writes over the same window for the same reason.LAST_USED_UPLOAD_REFRESH_INTERVAL = datetime.timedelta(hours=12)+DEADLOCK_RETRY_ATTEMPTS = 3+# The ID tiebreak keeps LIMIT/OFFSET paging stable when rows share a timestamp.# The reference is unique within a team, so it does not need a tiebreak.SYMBOL_SET_ORDERINGS: dict[str, tuple[str, ...]] = {@@ -192,11 +195,18 @@ def refresh_last_used(team: Team, chunk_ids: list[str]) -> None:symbolicated until a later deploy recreates the row."""now = timezone.now()- ErrorTrackingSymbolSet.objects.filter(- Q(last_used__isnull=True) | Q(last_used__lt=now - LAST_USED_UPLOAD_REFRESH_INTERVAL),- team=team,- ref__in=chunk_ids,- ).update(last_used=now)+ with transaction.atomic():+ stale_ids = list(+ ErrorTrackingSymbolSet.objects.filter(+ Q(last_used__isnull=True) | Q(last_used__lt=now - LAST_USED_UPLOAD_REFRESH_INTERVAL),+ team=team,+ ref__in=chunk_ids,+ )+ .order_by("ref")+ .select_for_update(no_key=True)+ .values_list("id", flat=True)+ )+ ErrorTrackingSymbolSet.objects.filter(team=team, id__in=stale_ids).update(last_used=now)def _validate_uploads(new_symbol_sets: list[SymbolSetUpload], team: Team) -> None:@@ -336,21 +346,30 @@ def bulk_check_symbol_sets(return chunk_ids_to_upload[email protected](capture_exceptions=False)-def bulk_create_symbol_sets(+def _lock_symbol_sets(team: Team, refs: list[str]) -> list[ErrorTrackingSymbolSet]:+ return list(+ ErrorTrackingSymbolSet.objects.filter(team=team, ref__in=refs).order_by("ref").select_for_update(no_key=True)+ )+++def _create_or_reuse_symbol_sets(new_symbol_sets: list[SymbolSetUpload],team: Team,- force: bool = False,- skip_on_conflict: bool = False,+ force: bool,+ skip_on_conflict: bool,) -> dict[str, dict[str, Any]]:- _validate_uploads(new_symbol_sets, team)- chunk_ids = [x.chunk_id for x in new_symbol_sets]+ chunk_ids = sorted(x.chunk_id for x in new_symbol_sets)id_url_map: dict[str, dict[str, Any]] = {}new_symbol_set_map = {x.chunk_id: x for x in new_symbol_sets}def reissue_upload(existing: ErrorTrackingSymbolSet) -> None:- storage_ptr = generate_symbol_set_file_key()+ # Pending uploads share an object key so a retry cannot invalidate an in-flight upload URL.+ storage_ptr = (+ existing.storage_ptr+ if existing.content_hash is None and existing.storage_ptr+ else generate_symbol_set_file_key()+ )id_url_map[existing.ref] = {**generate_symbol_set_upload_presigned_urls(storage_ptr, new_symbol_set_map[existing.ref].content_length),"symbol_set_id": str(existing.id),@@ -360,9 +379,9 @@ def bulk_create_symbol_sets(existing.content_hash = Nonewith transaction.atomic():- existing_symbol_sets = list(ErrorTrackingSymbolSet.objects.filter(team=team, ref__in=chunk_ids))- existing_symbol_set_refs = [s.ref for s in existing_symbol_sets]- missing_sets = list(set(chunk_ids) - set(existing_symbol_set_refs))+ existing_symbol_sets = _lock_symbol_sets(team, chunk_ids)+ existing_symbol_set_refs = {s.ref for s in existing_symbol_sets}+ missing_sets = [chunk_id for chunk_id in chunk_ids if chunk_id not in existing_symbol_set_refs]symbol_sets_to_be_created = []for chunk_id in missing_sets:@@ -382,10 +401,15 @@ def bulk_create_symbol_sets(symbol_sets_to_be_created.append(to_create)# create missing symbol sets- created_symbol_sets = ErrorTrackingSymbolSet.objects.bulk_create(symbol_sets_to_be_created)+ ErrorTrackingSymbolSet.objects.bulk_create(symbol_sets_to_be_created, ignore_conflicts=True)+ storage_ptrs_we_created = {s.ref: s.storage_ptr for s in symbol_sets_to_be_created}- for symbol_set in created_symbol_sets:- id_url_map[symbol_set.ref]["symbol_set_id"] = str(symbol_set.pk)+ for symbol_set in _lock_symbol_sets(team, missing_sets):+ if symbol_set.storage_ptr == storage_ptrs_we_created[symbol_set.ref]:+ id_url_map[symbol_set.ref]["symbol_set_id"] = str(symbol_set.pk)+ else:+ del id_url_map[symbol_set.ref]+ existing_symbol_sets.append(symbol_set)# update existing symbol setsto_update = []@@ -413,6 +437,24 @@ def bulk_create_symbol_sets(return id_url_map[email protected](capture_exceptions=False)+def bulk_create_symbol_sets(+ new_symbol_sets: list[SymbolSetUpload],+ team: Team,+ force: bool = False,+ skip_on_conflict: bool = False,+) -> dict[str, dict[str, Any]]:+ _validate_uploads(new_symbol_sets, team)+ for attempt in range(1, DEADLOCK_RETRY_ATTEMPTS):+ try:+ return _create_or_reuse_symbol_sets(new_symbol_sets, team, force, skip_on_conflict)+ except OperationalError as e:+ if not isinstance(e.__cause__, psycopg.errors.DeadlockDetected):+ raise+ logger.warning("symbol_set_upload_deadlock_retry", team_id=team.id, attempt=attempt)+ return _create_or_reuse_symbol_sets(new_symbol_sets, team, force, skip_on_conflict)++def list_symbol_sets(team_id: int,*,@@ -629,7 +671,9 @@ def bulk_finish_upload(content_hash = content_hashes[str(symbol_set.id)]symbol_set.content_hash = content_hash- ErrorTrackingSymbolSet.objects.bulk_update(symbol_sets, ["content_hash"])+ with transaction.atomic():+ _lock_symbol_sets(team, [symbol_set.ref for symbol_set in symbol_sets])+ ErrorTrackingSymbolSet.objects.bulk_update(symbol_sets, ["content_hash"])except Exception as e:posthoganalytics.capture("error_tracking_symbol_set_uploaded",diff --git a/products/error_tracking/backend/tests/api/test_error_tracking_api.py b/products/error_tracking/backend/tests/api/test_error_tracking_api.pyindex 6cc84c7315af..a6566790dd04 100644--- a/products/error_tracking/backend/tests/api/test_error_tracking_api.py+++ b/products/error_tracking/backend/tests/api/test_error_tracking_api.py@@ -2,17 +2,19 @@ import osimport jsonfrom collections.abc import Callablefrom datetime import UTC, datetime, timedelta+from typing import Anyfrom urllib.parse import unquoteimport time_machinefrom posthog.test.base import APIBaseTestfrom unittest.mock import ANY, Mock, patch-from django.db import connection+from django.db import OperationalError, connectionfrom django.test import SimpleTestCasefrom django.test.utils import CaptureQueriesContextfrom django.utils import timezone+import psycopgfrom boto3 import resourcefrom botocore.config import Configfrom parameterized import parameterized@@ -36,6 +38,7 @@ from products.error_tracking.backend.facade import (contracts,summaries as summaries_facade,)+from products.error_tracking.backend.logic import symbol_sets as symbol_sets_logicfrom products.error_tracking.backend.models import (ErrorTrackingAlert,ErrorTrackingAlertThread,@@ -1693,6 +1696,107 @@ class TestErrorTracking(APIBaseTest):assert "fallback_presigned_url" not in symbol_set_upload_responseassert symbol_set.last_used is not None+ @parameterized.expand(+ [+ ("pending", None, True),+ ("placeholder", None, True),+ ("uploaded", None, False),+ ("content_mismatch", "content_hash_mismatch", False),+ ("release_mismatch", "release_id_mismatch", False),+ ]+ )+ def test_bulk_start_upload_reuses_symbol_set_created_by_concurrent_upload(+ self, existing_state: str, expected_code: str | None, expect_upload: bool+ ) -> None:+ chunk_id = str(uuid7())+ new_chunk_id = str(uuid7())+ release = ErrorTrackingRelease.objects.create(+ team=self.team, hash_id="upload-release", version="1.0.0", project="test"+ )+ other_release = ErrorTrackingRelease.objects.create(+ team=self.team, hash_id="other-upload-release", version="2.0.0", project="test"+ )+ inserted = False+ winner: ErrorTrackingSymbolSet | None = None++ def insert_before_bulk_create(+ execute: Callable[..., object], sql: str, params: object, many: bool, context: object+ ) -> object:+ nonlocal inserted, winner+ if not inserted and sql.startswith('INSERT INTO "posthog_errortrackingsymbolset"'):+ inserted = True+ winner = ErrorTrackingSymbolSet.objects.create(+ team=self.team,+ ref=chunk_id,+ storage_ptr=None if existing_state == "placeholder" else "symbolsets/concurrent-upload",+ content_hash=("hash" if existing_state == "uploaded" else "other_hash" if expected_code else None),+ release=other_release if existing_state == "release_mismatch" else None,+ )+ return execute(sql, params, many, context)++ with connection.execute_wrapper(insert_before_bulk_create):+ response = self.client.post(+ f"/api/environments/{self.team.id}/error_tracking/symbol_sets/bulk_start_upload",+ data={+ "symbol_sets": [+ {"chunk_id": chunk_id, "content_hash": "hash", "release_id": str(release.id)},+ {"chunk_id": new_chunk_id, "content_hash": "new_hash"},+ ]+ },+ format="json",+ )++ assert winner is not None+ if expected_code:+ assert response.status_code == status.HTTP_400_BAD_REQUEST+ assert response.json()["code"] == expected_code+ assert not ErrorTrackingSymbolSet.objects.filter(team=self.team, ref=new_chunk_id).exists()+ return++ assert response.status_code == status.HTTP_201_CREATED+ id_map = response.json()["id_map"]+ symbol_set = ErrorTrackingSymbolSet.objects.get(team=self.team, ref=chunk_id)+ assert symbol_set.id == winner.id+ assert symbol_set.release_id == release.id+ assert (chunk_id in id_map) == expect_upload+ if expect_upload:+ assert id_map[chunk_id]["symbol_set_id"] == str(winner.id)+ assert id_map[chunk_id]["presigned_url"]["fields"]["key"] == symbol_set.storage_ptr+ if existing_state == "pending":+ assert symbol_set.storage_ptr == winner.storage_ptr+ else:+ assert symbol_set.content_hash == "hash"+ assert symbol_set.storage_ptr == winner.storage_ptr+ new_symbol_set = ErrorTrackingSymbolSet.objects.get(team=self.team, ref=new_chunk_id)+ assert id_map[new_chunk_id]["symbol_set_id"] == str(new_symbol_set.id)+ assert id_map[new_chunk_id]["presigned_url"]["fields"]["key"] == new_symbol_set.storage_ptr++ def test_bulk_start_upload_retries_after_deadlock(self) -> None:+ chunk_id = str(uuid7())+ deadlock = OperationalError("deadlock detected")+ deadlock.__cause__ = psycopg.errors.DeadlockDetected("deadlock detected")+ create_or_reuse = symbol_sets_logic._create_or_reuse_symbol_sets+ calls = 0++ def deadlock_on_first_attempt(*args: Any) -> dict[str, dict[str, Any]]:+ nonlocal calls+ calls += 1+ if calls == 1:+ raise deadlock+ return create_or_reuse(*args)++ with patch.object(symbol_sets_logic, "_create_or_reuse_symbol_sets", side_effect=deadlock_on_first_attempt):+ response = self.client.post(+ f"/api/environments/{self.team.id}/error_tracking/symbol_sets/bulk_start_upload",+ data={"chunk_ids": [chunk_id]},+ )++ assert response.status_code == status.HTTP_201_CREATED+ assert calls == 2+ assert response.json()["id_map"][chunk_id]["symbol_set_id"] == str(+ ErrorTrackingSymbolSet.objects.get(team=self.team, ref=chunk_id).id+ )+def test_bulk_start_upload_includes_fallback_presigned_url_when_accelerated(self) -> None:chunk_id = str(uuid7())with (@@ -2060,8 +2164,9 @@ class TestErrorTracking(APIBaseTest):symbol_set.refresh_from_db()assert symbol_set.release_id == release.id- assert symbol_set.storage_ptr != initial_storage_ptr+ assert symbol_set.storage_ptr == initial_storage_ptrassert id_map[str(chunk_id)]["symbol_set_id"] == str(symbol_set.id)+ assert id_map[str(chunk_id)]["presigned_url"]["fields"]["key"] == initial_storage_ptrdef test_bulk_start_upload_updates_release_for_uploaded_symbol_set(self) -> None:chunk_id = str(uuid7())@@ -2128,8 +2233,9 @@ class TestErrorTracking(APIBaseTest):id_map = second_response.json()["id_map"]symbol_set.refresh_from_db()- assert symbol_set.storage_ptr != initial_storage_ptr+ assert symbol_set.storage_ptr == initial_storage_ptrassert id_map[str(chunk_id)]["symbol_set_id"] == str(symbol_set.id)+ assert id_map[str(chunk_id)]["presigned_url"]["fields"]["key"] == initial_storage_ptrdef test_bulk_start_upload_rejects_release_change(self) -> None:chunk_id = str(uuid7())