Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(error-tracking): distinguish skipped symbol set uploads (#121058)

Hugues Pouillotecfc9561b36d
diff --git a/cli/.sampo/changesets/sullen-thunderbearer-aurelien.md b/cli/.sampo/changesets/sullen-thunderbearer-aurelien.md
new file mode 100644
index 000000000000..1781e9f3f545
--- /dev/null
+++ b/cli/.sampo/changesets/sullen-thunderbearer-aurelien.md
@@ -0,0 +1,5 @@
+---
+cargo/posthog-cli: patch
+---
+
+Correlate symbol set upload batches within a CLI run
diff --git a/cli/src/api/symbol_sets.rs b/cli/src/api/symbol_sets.rs
index a3007178a38b..3e1f93d0343e 100644
--- a/cli/src/api/symbol_sets.rs
+++ b/cli/src/api/symbol_sets.rs
@@ -171,6 +171,10 @@ pub struct PresignedUrl {
#[derive(Debug, Clone, Serialize, Deserialize)]
struct BulkUploadRequest {
symbol_sets: Vec<CreateSymbolSetRequest>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ upload_attempt_id: Option<String>,
+ #[serde(skip_serializing_if = "Option::is_none")]
+ upload_run_id: Option<String>,
/// When true, allow overwriting symbol sets whose content has changed.
#[serde(default)]
force: bool,
@@ -198,6 +202,8 @@ struct HashedUpload<'a> {
#[derive(Debug, Clone, Serialize, Deserialize)]
struct BulkUploadFinishRequest {
content_hashes: HashMap<String, String>,
+ upload_attempt_id: String,
+ upload_run_id: String,
}
/// Upload symbol sets with optional retry on release_id_mismatch error.
@@ -246,6 +252,8 @@ pub fn upload_with_retry_and_concurrency(
Ok(client) => UploadTransport {
client,
accelerated_unreachable: AtomicBool::new(false),
+ // A release-ID retry reuses this transport and belongs to the same CLI run.
+ upload_run_id: uuid::Uuid::now_v7().to_string(),
},
Err(e) => return (summary, Err(e)),
};
@@ -305,13 +313,13 @@ fn build_upload_thread_pool(concurrency: NonZeroUsize) -> Result<ThreadPool> {
.context("Failed to initialize symbol set upload thread pool")
}
-/// Shared S3 upload transport for one run: a single client, because reusing its
-/// connection pool avoids paying a TCP + TLS handshake per uploaded chunk, and
-/// the run-wide latch that stops attempts against the accelerated endpoint once
-/// it is known to be unreachable.
+/// Shared S3 upload transport and tracking ID for one run. Reusing the client avoids
+/// a TCP + TLS handshake per chunk; the run-wide latch stops attempts against the
+/// accelerated endpoint once it is known to be unreachable.
struct UploadTransport {
client: Client,
accelerated_unreachable: AtomicBool,
+ upload_run_id: String,
}
impl UploadTransport {
@@ -381,7 +389,14 @@ fn upload_inner(
for (i, batch) in to_upload.chunks(batch_size).enumerate() {
info!("Starting upload of batch {i}, {} symbol sets", batch.len());
- let start_response = start_upload(batch, force, skip_on_conflict)?;
+ let upload_attempt_id = uuid::Uuid::now_v7().to_string();
+ let start_response = start_upload(
+ batch,
+ force,
+ skip_on_conflict,
+ &upload_attempt_id,
+ &transport.upload_run_id,
+ )?;
let id_map: HashMap<&str, HashedUpload> = batch
.iter()
@@ -418,7 +433,7 @@ fn upload_inner(
}
let uploaded = content_hashes.len();
- finish_upload(content_hashes)?;
+ finish_upload(content_hashes, &upload_attempt_id, &transport.upload_run_id)?;
summary.uploaded += uploaded;
}
@@ -476,6 +491,8 @@ fn bulk_upload_request(
skip_on_conflict: bool,
) -> BulkUploadRequest {
BulkUploadRequest {
+ upload_attempt_id: None,
+ upload_run_id: None,
symbol_sets: batch
.iter()
.map(|hashed| CreateSymbolSetRequest {
@@ -544,9 +561,15 @@ fn start_upload(
batch: &[HashedUpload],
force: bool,
skip_on_conflict: bool,
+ upload_attempt_id: &str,
+ upload_run_id: &str,
) -> Result<BulkUploadStartResponse, UploadError> {
let client = &context().client;
- let request = bulk_upload_request(batch, force, skip_on_conflict);
+ let request = BulkUploadRequest {
+ upload_attempt_id: Some(upload_attempt_id.to_string()),
+ upload_run_id: Some(upload_run_id.to_string()),
+ ..bulk_upload_request(batch, force, skip_on_conflict)
+ };
let res = retry(retry_policy(500, 2, 3), |_| {
client.send_post(
@@ -686,9 +709,17 @@ fn upload_to_s3(
Ok(())
}
-fn finish_upload(content_hashes: HashMap<String, String>) -> Result<(), UploadError> {
+fn finish_upload(
+ content_hashes: HashMap<String, String>,
+ upload_attempt_id: &str,
+ upload_run_id: &str,
+) -> Result<(), UploadError> {
let client = &context().client;
- let request = BulkUploadFinishRequest { content_hashes };
+ let request = BulkUploadFinishRequest {
+ content_hashes,
+ upload_attempt_id: upload_attempt_id.to_string(),
+ upload_run_id: upload_run_id.to_string(),
+ };
retry(retry_policy(500, 2, 3), |_| {
client.send_post(
@@ -1063,6 +1094,39 @@ mod tests {
}
}
+ #[test]
+ fn upload_ids_correlate_batches_and_finishes_but_not_checks() {
+ let check = bulk_upload_request(&[], false, false);
+ let check_json = serde_json::to_value(&check).unwrap();
+ assert!(check_json.get("upload_attempt_id").is_none());
+ assert!(check_json.get("upload_run_id").is_none());
+
+ let upload_run_id = uuid::Uuid::now_v7().to_string();
+ let attempt_ids = [
+ uuid::Uuid::now_v7().to_string(),
+ uuid::Uuid::now_v7().to_string(),
+ ];
+ assert_ne!(attempt_ids[0], attempt_ids[1]);
+ for upload_attempt_id in attempt_ids {
+ let start = BulkUploadRequest {
+ upload_attempt_id: Some(upload_attempt_id.clone()),
+ upload_run_id: Some(upload_run_id.clone()),
+ ..check.clone()
+ };
+ let finish = BulkUploadFinishRequest {
+ content_hashes: HashMap::new(),
+ upload_attempt_id: upload_attempt_id.clone(),
+ upload_run_id: upload_run_id.clone(),
+ };
+ let start_json = serde_json::to_value(&start).unwrap();
+ let finish_json = serde_json::to_value(&finish).unwrap();
+ assert_eq!(start_json["upload_attempt_id"], upload_attempt_id);
+ assert_eq!(finish_json["upload_attempt_id"], upload_attempt_id);
+ assert_eq!(start_json["upload_run_id"], upload_run_id);
+ assert_eq!(finish_json["upload_run_id"], upload_run_id);
+ }
+ }
+
#[test]
fn start_upload_response_parses_without_fallback_presigned_url() {
let json = r#"{"id_map":{"chunk":{"presigned_url":{"url":"https://example.com/","fields":{}},"symbol_set_id":"id"}}}"#;
@@ -1160,6 +1224,7 @@ mod tests {
UploadTransport {
client: Client::new(),
accelerated_unreachable: AtomicBool::new(false),
+ upload_run_id: uuid::Uuid::now_v7().to_string(),
}
}
@@ -1262,10 +1327,11 @@ mod tests {
crate::invocation_context::InvocationContext::new(config, client)
});
- let err = finish_upload(HashMap::from([(
- "symbol-set-id".to_string(),
- "content-hash".to_string(),
- )]))
+ let err = finish_upload(
+ HashMap::from([("symbol-set-id".to_string(), "content-hash".to_string())]),
+ "test-attempt",
+ "test-run",
+ )
.unwrap_err();
let UploadError::Other(err) = err else {
panic!("expected UploadError::Other");
diff --git a/posthog/event_usage.py b/posthog/event_usage.py
index fe5055ac0efc..ecc3076aff2e 100644
--- a/posthog/event_usage.py
+++ b/posthog/event_usage.py
@@ -437,6 +437,8 @@ AnalyticsProps = TypedDict(
"mcp_client_version": NotRequired[str | None],
"mcp_protocol_version": NotRequired[str | None],
"mcp_oauth_client_name": NotRequired[str | None],
+ "upload_attempt_id": NotRequired[str],
+ "upload_run_id": NotRequired[str],
},
total=False,
)
diff --git a/products/error_tracking/backend/presentation/views/symbol_sets.py b/products/error_tracking/backend/presentation/views/symbol_sets.py
index 980697f9c060..def5cc7c0097 100644
--- a/products/error_tracking/backend/presentation/views/symbol_sets.py
+++ b/products/error_tracking/backend/presentation/views/symbol_sets.py
@@ -100,6 +100,14 @@ class ErrorTrackingSymbolSetBulkCheckUploadResponseSerializer(serializers.Serial
class ErrorTrackingSymbolSetBulkStartUploadSerializer(ErrorTrackingSymbolSetBulkCheckUploadSerializer):
+ upload_attempt_id = serializers.UUIDField(
+ required=False,
+ help_text="Optional ID used to correlate upload analytics across start and finish requests.",
+ )
+ upload_run_id = serializers.UUIDField(
+ required=False,
+ help_text="Optional ID shared by every batch in one CLI upload run, including release-ID retries.",
+ )
chunk_ids = serializers.ListField(
child=serializers.CharField(),
required=False,
@@ -153,6 +161,14 @@ class ErrorTrackingSymbolSetBulkStartUploadResponseSerializer(serializers.Serial
class ErrorTrackingSymbolSetBulkFinishUploadSerializer(serializers.Serializer):
+ upload_attempt_id = serializers.UUIDField(
+ required=False,
+ help_text="Optional ID from the corresponding bulk start request for upload analytics.",
+ )
+ upload_run_id = serializers.UUIDField(
+ required=False,
+ help_text="Optional ID shared by every batch in one CLI upload run, including release-ID retries.",
+ )
content_hashes = serializers.DictField(
child=serializers.CharField(),
help_text="Map of symbol set ID to uploaded content hash.",
@@ -436,7 +452,7 @@ class ErrorTrackingSymbolSetViewSet(TeamAndOrgViewSetMixin, viewsets.GenericView
# get no entry in the id_map, so its size is the number of chunks being uploaded.
total_chunks = len(symbol_sets) + len(chunk_ids)
posthoganalytics.capture(
- "error_tracking_symbol_set_upload_started",
+ "error_tracking_symbol_set_upload_started" if id_map else "error_tracking_symbol_set_upload_skipped",
properties={
**get_request_analytics_properties(request),
"team_id": self.team.id,
@@ -445,6 +461,13 @@ class ErrorTrackingSymbolSetViewSet(TeamAndOrgViewSetMixin, viewsets.GenericView
"skip_on_conflict": skip_on_conflict,
"total_chunks": total_chunks,
"chunks_skipped": total_chunks - len(id_map),
+ "chunks_to_upload": len(id_map),
+ **(
+ {"upload_attempt_id": str(upload_data["upload_attempt_id"])}
+ if "upload_attempt_id" in upload_data
+ else {}
+ ),
+ **({"upload_run_id": str(upload_data["upload_run_id"])} if "upload_run_id" in upload_data else {}),
},
groups=groups(self.team.organization, self.team),
)
@@ -459,14 +482,27 @@ class ErrorTrackingSymbolSetViewSet(TeamAndOrgViewSetMixin, viewsets.GenericView
if content_hashes is None:
return Response({"detail": "content_hashes are required"}, status=status.HTTP_400_BAD_REQUEST)
+ finish_data = {"content_hashes": content_hashes}
+ if "upload_attempt_id" in request.data:
+ finish_data["upload_attempt_id"] = request.data["upload_attempt_id"]
+ if "upload_run_id" in request.data:
+ finish_data["upload_run_id"] = request.data["upload_run_id"]
+ finish_serializer = ErrorTrackingSymbolSetBulkFinishUploadSerializer(data=finish_data)
+ finish_serializer.is_valid(raise_exception=True)
+ upload_attempt_id = finish_serializer.validated_data.get("upload_attempt_id")
+ upload_run_id = finish_serializer.validated_data.get("upload_run_id")
+
if len(content_hashes) == 0:
# This can happen if someone re-runs an upload against a directory that's already been
# uploaded - we'll return no new upload keys, they'll upload nothing, and then
# we can early exit here.
return Response({"success": True}, status=status.HTTP_201_CREATED)
- symbol_sets_facade.bulk_finish_upload(
- self.team, content_hashes, analytics_props=get_request_analytics_properties(request)
- )
+ analytics_props = get_request_analytics_properties(request)
+ if upload_attempt_id:
+ analytics_props["upload_attempt_id"] = str(upload_attempt_id)
+ if upload_run_id:
+ analytics_props["upload_run_id"] = str(upload_run_id)
+ symbol_sets_facade.bulk_finish_upload(self.team, content_hashes, analytics_props=analytics_props)
return Response({"success": True}, status=status.HTTP_201_CREATED)
diff --git a/products/error_tracking/backend/tests/api/test_error_tracking_api.py b/products/error_tracking/backend/tests/api/test_error_tracking_api.py
index 3c492f35a3fc..9c41ef1bef1c 100644
--- a/products/error_tracking/backend/tests/api/test_error_tracking_api.py
+++ b/products/error_tracking/backend/tests/api/test_error_tracking_api.py
@@ -1705,10 +1705,14 @@ class TestErrorTracking(APIBaseTest):
)
new_chunk_id = str(uuid7())
+ upload_attempt_id = str(uuid7())
+ upload_run_id = str(uuid7())
response = self.client.post(
f"/api/environments/{self.team.id}/error_tracking/symbol_sets/bulk_start_upload",
data={
+ "upload_attempt_id": upload_attempt_id,
+ "upload_run_id": upload_run_id,
"symbol_sets": [
{
"chunk_id": existing_chunk_id,
@@ -1720,7 +1724,7 @@ class TestErrorTracking(APIBaseTest):
"release_id": str(release.id),
"content_hash": "new_hash",
},
- ]
+ ],
},
format="json",
)
@@ -1751,6 +1755,9 @@ class TestErrorTracking(APIBaseTest):
"skip_on_conflict": False,
"total_chunks": 2,
"chunks_skipped": 1,
+ "chunks_to_upload": 1,
+ "upload_attempt_id": upload_attempt_id,
+ "upload_run_id": upload_run_id,
}
@parameterized.expand(
@@ -1760,8 +1767,9 @@ class TestErrorTracking(APIBaseTest):
("used_within_the_refresh_interval", timedelta(hours=1), False),
]
)
+ @patch("products.error_tracking.backend.presentation.views.symbol_sets.posthoganalytics.capture")
def test_bulk_start_upload_marks_unchanged_symbol_sets_as_used(
- self, _name: str, last_used_age: timedelta | None, expect_refresh: bool
+ self, _name: str, last_used_age: timedelta | None, expect_refresh: bool, patched_capture: Mock
) -> None:
chunk_id = str(uuid7())
last_used = None if last_used_age is None else timezone.now() - last_used_age
@@ -1774,15 +1782,23 @@ class TestErrorTracking(APIBaseTest):
)
before_upload = timezone.now()
+ upload_run_id = str(uuid7())
response = self.client.post(
f"/api/environments/{self.team.id}/error_tracking/symbol_sets/bulk_start_upload",
- data={"symbol_sets": [{"chunk_id": chunk_id, "content_hash": "already_uploaded"}]},
+ data={
+ "upload_run_id": upload_run_id,
+ "symbol_sets": [{"chunk_id": chunk_id, "content_hash": "already_uploaded"}],
+ },
format="json",
)
after_upload = timezone.now()
assert response.status_code == status.HTTP_201_CREATED
assert response.json()["id_map"] == {}
+ assert patched_capture.call_args.args[0] == "error_tracking_symbol_set_upload_skipped"
+ assert patched_capture.call_args.kwargs["properties"]["chunks_to_upload"] == 0
+ assert patched_capture.call_args.kwargs["properties"]["chunks_skipped"] == 1
+ assert patched_capture.call_args.kwargs["properties"]["upload_run_id"] == upload_run_id
symbol_set.refresh_from_db()
if expect_refresh:
@@ -2296,7 +2312,13 @@ class TestErrorTracking(APIBaseTest):
self, patched_object_storage, patched_capture: Mock
) -> None:
symbol_set = ErrorTrackingSymbolSet.objects.create(team=self.team, ref=str(uuid7()), storage_ptr="file/name")
- request_data = {"content_hashes": {str(symbol_set.id): "hash"}}
+ upload_attempt_id = str(uuid7())
+ upload_run_id = str(uuid7())
+ request_data = {
+ "content_hashes": {str(symbol_set.id): "hash"},
+ "upload_attempt_id": upload_attempt_id,
+ "upload_run_id": upload_run_id,
+ }
patched_object_storage.side_effect = [None, {"ContentLength": 1000}]
@@ -2318,6 +2340,8 @@ class TestErrorTracking(APIBaseTest):
"file_count": 1,
"failure_reason": "ValidationError",
"failure_code": "file_not_found",
+ "upload_attempt_id": upload_attempt_id,
+ "upload_run_id": upload_run_id,
}
symbol_set.refresh_from_db()
@@ -2330,6 +2354,9 @@ class TestErrorTracking(APIBaseTest):
)
assert retry_response.status_code == status.HTTP_201_CREATED
+ assert patched_capture.call_args.kwargs["properties"]["upload_attempt_id"] == upload_attempt_id
+ assert patched_capture.call_args.kwargs["properties"]["upload_run_id"] == upload_run_id
+ assert patched_capture.call_args.kwargs["properties"]["success"] is True
symbol_set.refresh_from_db()
assert symbol_set.content_hash == "hash"
diff --git a/products/error_tracking/frontend/generated/api.schemas.ts b/products/error_tracking/frontend/generated/api.schemas.ts
index faf793740105..9285fa2de52f 100644
--- a/products/error_tracking/frontend/generated/api.schemas.ts
+++ b/products/error_tracking/frontend/generated/api.schemas.ts
@@ -2318,6 +2318,10 @@ export interface ErrorTrackingSymbolSetBulkDeleteApi {
export type ErrorTrackingSymbolSetBulkFinishUploadApiContentHashes = { [key: string]: string }
export interface ErrorTrackingSymbolSetBulkFinishUploadApi {
+ /** Optional ID from the corresponding bulk start request for upload analytics. */
+ upload_attempt_id?: string
+ /** Optional ID shared by every batch in one CLI upload run, including release-ID retries. */
+ upload_run_id?: string
/** Map of symbol set ID to uploaded content hash. */
content_hashes: ErrorTrackingSymbolSetBulkFinishUploadApiContentHashes
}
@@ -2329,6 +2333,10 @@ export interface ErrorTrackingSymbolSetBulkStartUploadApi {
force?: boolean
/** Whether to skip uploaded symbol sets whose content hash changed instead of failing. */
skip_on_conflict?: boolean
+ /** Optional ID used to correlate upload analytics across start and finish requests. */
+ upload_attempt_id?: string
+ /** Optional ID shared by every batch in one CLI upload run, including release-ID retries. */
+ upload_run_id?: string
/** Legacy list of symbol set references to upload, all associated with `release_id`. */
chunk_ids?: string[]
/**
diff --git a/products/error_tracking/frontend/generated/api.zod.ts b/products/error_tracking/frontend/generated/api.zod.ts
index 8bf077731be3..7258c349966d 100644
--- a/products/error_tracking/frontend/generated/api.zod.ts
+++ b/products/error_tracking/frontend/generated/api.zod.ts
@@ -1312,6 +1312,14 @@ export const ErrorTrackingSymbolSetsBulkDeleteCreateBody = /* @__PURE__ */ zod.o
})
export const ErrorTrackingSymbolSetsBulkFinishUploadCreateBody = /* @__PURE__ */ zod.object({
+ upload_attempt_id: zod
+ .uuid()
+ .optional()
+ .describe('Optional ID from the corresponding bulk start request for upload analytics.'),
+ upload_run_id: zod
+ .uuid()
+ .optional()
+ .describe('Optional ID shared by every batch in one CLI upload run, including release-ID retries.'),
content_hashes: zod.record(zod.string(), zod.string()).describe('Map of symbol set ID to uploaded content hash.'),
})
@@ -1352,6 +1360,14 @@ export const ErrorTrackingSymbolSetsBulkStartUploadCreateBody = /* @__PURE__ */
.boolean()
.default(errorTrackingSymbolSetsBulkStartUploadCreateBodySkipOnConflictDefault)
.describe('Whether to skip uploaded symbol sets whose content hash changed instead of failing.'),
+ upload_attempt_id: zod
+ .uuid()
+ .optional()
+ .describe('Optional ID used to correlate upload analytics across start and finish requests.'),
+ upload_run_id: zod
+ .uuid()
+ .optional()
+ .describe('Optional ID shared by every batch in one CLI upload run, including release-ID retries.'),
chunk_ids: zod
.array(zod.string())
.optional()
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index f0264b437af9..abe752ab64ad 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -42299,6 +42299,10 @@ export namespace Schemas {
export type ErrorTrackingSymbolSetBulkFinishUploadContentHashes = {[key: string]: string};
export interface ErrorTrackingSymbolSetBulkFinishUpload {
+ /** Optional ID from the corresponding bulk start request for upload analytics. */
+ upload_attempt_id?: string;
+ /** Optional ID shared by every batch in one CLI upload run, including release-ID retries. */
+ upload_run_id?: string;
/** Map of symbol set ID to uploaded content hash. */
content_hashes: ErrorTrackingSymbolSetBulkFinishUploadContentHashes;
}
@@ -42310,6 +42314,10 @@ export namespace Schemas {
force?: boolean;
/** Whether to skip uploaded symbol sets whose content hash changed instead of failing. */
skip_on_conflict?: boolean;
+ /** Optional ID used to correlate upload analytics across start and finish requests. */
+ upload_attempt_id?: string;
+ /** Optional ID shared by every batch in one CLI upload run, including release-ID retries. */
+ upload_run_id?: string;
/** Legacy list of symbol set references to upload, all associated with `release_id`. */
chunk_ids?: string[];
/**