:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): retry a transient S3 error when deleting a staged CDP chunk (#117491)
Tom Owersdcf358f5ba3d
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.pyindex 0169b45697a5..68cab77d39c2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py@@ -33,6 +33,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdpemitted_rows_key,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store import (+ aretry_staged_delete,aretry_staged_read,aretry_staged_write,)@@ -392,7 +393,11 @@ class CDPProducer:CDP_PRODUCER_ROWS_TOTAL.labels(team_id=str(self.team_id)).inc(row_index)await self.logger.adebug(f"Produced {row_index} rows")await self.logger.adebug(f"Deleting file {file_path}")- await asyncio.to_thread(fs.delete_file, file_path)++ async def _delete_staged_file(path: str = file_path) -> None:+ await asyncio.to_thread(fs.delete_file, path)++ await aretry_staged_delete(_delete_staged_file, path=file_path, logger=self.logger)await self.logger.adebug("Finished producing all CDP data to Kafka")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.pyindex caa8272be129..0d1f5910deed 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/staging_object_store.py@@ -103,3 +103,15 @@ async def aretry_staged_read(and produces no rows, so a retried open can't duplicate anything a prior attempt already sent."""return await _aretry_staged_operation(operation, verb="open", path=path, logger=logger)+++async def aretry_staged_delete(+ operation: Callable[[], Coroutine[Any, Any, None]], *, path: str, logger: FilteringBoundLogger+) -> None:+ """Delete one staged file, retrying a transient object-store failure with backoff.++ Pass a zero-arg callable that produces the awaitable, not the awaitable itself, so a retry can+ reissue the delete. Safe to retry: DeleteObject is idempotent, and a staged file is only ever+ deleted after it has already been fully consumed.+ """+ await _aretry_staged_operation(operation, verb="delete", path=path, logger=logger)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.pyindex 1b6bcf70f6d3..56b8f4b6c2c4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py@@ -680,6 +680,74 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_read_failure(mock_capture_exception.assert_not_called()[email protected]_db(transaction=True)+@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")+@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.capture_exception")+async def test_produce_to_kafka_from_s3_retries_a_transient_s3_delete_failure(+ mock_capture_exception, mock_get_s3_client, team+):+ # S3's SlowDown throttling response on DeleteObject is a known-transient blip (see+ # is_transient_object_store_error), not a bug in our code: unlike the read and write paths,+ # the post-produce cleanup delete used to call fs.delete_file directly with no retry, so a+ # single SlowDown response escaped produce_to_kafka_from_s3 uncaught and was reported as a+ # fresh error-tracking issue instead of being retried.+ source = await sync_to_async(ExternalDataSource.objects.create)(+ team=team, source_type=ExternalDataSourceType.POSTGRES+ )+ table = await sync_to_async(DataWarehouseTable.objects.create)(+ team=team, name="postgres_table_1", external_data_source=source+ )+ schema = await sync_to_async(ExternalDataSchema.objects.create)(+ team=team, name="table_1", source=source, table=table+ )++ mock_s3_client = mock.AsyncMock()+ mock_s3_client._ls.return_value = [{"Key": "path/chunk_0.parquet", "type": "file"}]+ mock_get_s3_client.return_value.__aenter__ = mock.AsyncMock(return_value=mock_s3_client)+ mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)++ mock_kafka_producer = MagicMock()+ mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.flush = mock.AsyncMock()+ mock_kafka_producer.close = mock.AsyncMock()++ test_data = pa.table({"id": [1], "name": ["Alice"]})+ parquet_buffer = BytesIO()+ pq.write_table(test_data, parquet_buffer, compression="zstd")+ parquet_buffer.seek(0)++ mock_file = MagicMock()+ mock_file.__enter__ = MagicMock(return_value=parquet_buffer)+ mock_file.__exit__ = MagicMock(return_value=False)++ transient_error = OSError(+ "When delete key 'chunk_0.parquet' in bucket 'example-bucket': "+ "AWS Error SLOW_DOWN during DeleteObject operation: Please reduce your request rate."+ )+ mock_fs = MagicMock()+ mock_fs.open_input_file.return_value = mock_file+ mock_fs.delete_file.side_effect = [transient_error, None]++ producer = CDPProducer.for_source(+ team_id=team.id, schema_id=str(schema.id), job_id="test_job", logger=mock.AsyncMock()+ )++ with (+ patch.object(producer, "_get_fs", return_value=mock_fs),+ _patch_async_producer_scope(mock_kafka_producer),+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store.asyncio.sleep",+ new=mock.AsyncMock(),+ ),+ ):+ await producer.produce_to_kafka_from_s3()++ assert mock_fs.delete_file.call_count == 2+ mock_kafka_producer.produce.assert_called_once()+ mock_capture_exception.assert_not_called()++@pytest.mark.django_db(transaction=True)@pytest.mark.asyncio@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")