Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): retry a transient S3 error when deleting a staged CDP chunk (#117491)

Tom Owersdcf358f5ba3d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
index 0169b45697a5..68cab77d39c2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp
emitted_rows_key,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store import (
+ aretry_staged_delete,
aretry_staged_read,
aretry_staged_write,
)
@@ -392,7 +393,11 @@ class CDPProducer:
CDP_PRODUCER_ROWS_TOTAL.labels(team_id=str(self.team_id)).inc(row_index)
await self.logger.adebug(f"Produced {row_index} rows")
await self.logger.adebug(f"Deleting file {file_path}")
- await asyncio.to_thread(fs.delete_file, file_path)
+
+ async def _delete_staged_file(path: str = file_path) -> None:
+ await asyncio.to_thread(fs.delete_file, path)
+
+ await aretry_staged_delete(_delete_staged_file, path=file_path, logger=self.logger)
await self.logger.adebug("Finished producing all CDP data to Kafka")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.py
index caa8272be129..0d1f5910deed 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.py
@@ -103,3 +103,15 @@ async def aretry_staged_read(
and produces no rows, so a retried open can't duplicate anything a prior attempt already sent.
"""
return await _aretry_staged_operation(operation, verb="open", path=path, logger=logger)
+
+
+async def aretry_staged_delete(
+ operation: Callable[[], Coroutine[Any, Any, None]], *, path: str, logger: FilteringBoundLogger
+) -> None:
+ """Delete one staged file, retrying a transient object-store failure with backoff.
+
+ Pass a zero-arg callable that produces the awaitable, not the awaitable itself, so a retry can
+ reissue the delete. Safe to retry: DeleteObject is idempotent, and a staged file is only ever
+ deleted after it has already been fully consumed.
+ """
+ await _aretry_staged_operation(operation, verb="delete", path=path, logger=logger)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
index 1b6bcf70f6d3..56b8f4b6c2c4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
@@ -680,6 +680,74 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_read_failure(
mock_capture_exception.assert_not_called()
[email protected]_db(transaction=True)
+@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")
+@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.capture_exception")
+async def test_produce_to_kafka_from_s3_retries_a_transient_s3_delete_failure(
+ mock_capture_exception, mock_get_s3_client, team
+):
+ # S3's SlowDown throttling response on DeleteObject is a known-transient blip (see
+ # is_transient_object_store_error), not a bug in our code: unlike the read and write paths,
+ # the post-produce cleanup delete used to call fs.delete_file directly with no retry, so a
+ # single SlowDown response escaped produce_to_kafka_from_s3 uncaught and was reported as a
+ # fresh error-tracking issue instead of being retried.
+ source = await sync_to_async(ExternalDataSource.objects.create)(
+ team=team, source_type=ExternalDataSourceType.POSTGRES
+ )
+ table = await sync_to_async(DataWarehouseTable.objects.create)(
+ team=team, name="postgres_table_1", external_data_source=source
+ )
+ schema = await sync_to_async(ExternalDataSchema.objects.create)(
+ team=team, name="table_1", source=source, table=table
+ )
+
+ mock_s3_client = mock.AsyncMock()
+ mock_s3_client._ls.return_value = [{"Key": "path/chunk_0.parquet", "type": "file"}]
+ mock_get_s3_client.return_value.__aenter__ = mock.AsyncMock(return_value=mock_s3_client)
+ mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
+
+ mock_kafka_producer = MagicMock()
+ mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.flush = mock.AsyncMock()
+ mock_kafka_producer.close = mock.AsyncMock()
+
+ test_data = pa.table({"id": [1], "name": ["Alice"]})
+ parquet_buffer = BytesIO()
+ pq.write_table(test_data, parquet_buffer, compression="zstd")
+ parquet_buffer.seek(0)
+
+ mock_file = MagicMock()
+ mock_file.__enter__ = MagicMock(return_value=parquet_buffer)
+ mock_file.__exit__ = MagicMock(return_value=False)
+
+ transient_error = OSError(
+ "When delete key 'chunk_0.parquet' in bucket 'example-bucket': "
+ "AWS Error SLOW_DOWN during DeleteObject operation: Please reduce your request rate."
+ )
+ mock_fs = MagicMock()
+ mock_fs.open_input_file.return_value = mock_file
+ mock_fs.delete_file.side_effect = [transient_error, None]
+
+ producer = CDPProducer.for_source(
+ team_id=team.id, schema_id=str(schema.id), job_id="test_job", logger=mock.AsyncMock()
+ )
+
+ with (
+ patch.object(producer, "_get_fs", return_value=mock_fs),
+ _patch_async_producer_scope(mock_kafka_producer),
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store.asyncio.sleep",
+ new=mock.AsyncMock(),
+ ),
+ ):
+ await producer.produce_to_kafka_from_s3()
+
+ assert mock_fs.delete_file.call_count == 2
+ mock_kafka_producer.produce.assert_called_once()
+ mock_capture_exception.assert_not_called()
+
+
@pytest.mark.django_db(transaction=True)
@pytest.mark.asyncio
@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")