:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): stop retrying an S3 destination's invalid parameters forever (#119031)
Tom Owers3deeea24e532
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_s3_configuration_errors.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_s3_configuration_errors.pyindex d50d1234215c..1e7065f0d38d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_s3_configuration_errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_s3_configuration_errors.py@@ -1,5 +1,6 @@from __future__ import annotations+import dataclassesfrom collections.abc import AsyncIteratorfrom contextlib import asynccontextmanagerfrom typing import TYPE_CHECKING, Any@@ -83,6 +84,11 @@ class TestConfigurationErrors:(_client_error("AccessDenied", "AssumeRole"), "trust policy", True),(_client_error("AccessDeniedException", "AssumeRole"), "external ID", True),(InvalidCredentialsError("fake-secret"), "trust policy", True),+ (+ botocore.exceptions.ParamValidationError(report='Invalid bucket name "foo": fake-secret'),+ "bucket name, region and prefix",+ False,+ ),],)async def test_permanent_errors_are_configuration_errors(@@ -143,3 +149,15 @@ class TestConfigurationErrors:assert caught.value is errorassert not isinstance(caught.value, DestinationConfigurationError)++ async def test_bucket_name_whitespace_is_trimmed(self) -> None:+ # A bucket name pasted with a stray leading/trailing space fails S3's name regex with+ # no retry able to fix it. Trimming it here is the difference between that and a+ # working sync.+ ctx = dataclasses.replace(_ctx(), config={"bucket": " example-bucket ", "region": "us-east-1"})+ client = mock.AsyncMock()+ writer = LocalS3Writer(ctx, client, connection_error=None)++ await writer.write_batch(_batches(), DestinationBatchContext(run=ctx, batch_index=0, is_final_batch=False))++ assert client.put_object.call_args.kwargs["Bucket"] == "example-bucket"diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.pyindex f4b173f6e2a2..cce0706d86b8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/s3.py@@ -96,11 +96,18 @@ ROLE_DETAIL = ("PostHog cannot assume the role. Check the role ARN and the trust policy with the external ID, ""then run the sync again.")+INVALID_PARAMS_DETAIL = (+ "The destination's S3 configuration is invalid. Check the bucket name, region and prefix, then run the sync again."+)def _configuration_error_detail(err: Exception) -> str | None:if isinstance(err, InvalidCredentialsError):return ROLE_DETAIL+ if isinstance(err, botocore.exceptions.ParamValidationError):+ # Raised by botocore itself, before any request reaches S3, so a retry sends the exact+ # same invalid parameters and fails identically every time.+ return INVALID_PARAMS_DETAILif not isinstance(err, botocore.exceptions.ClientError):return Noneif err.response.get("ResponseMetadata", {}).get("HTTPStatusCode", 0) >= 500:@@ -286,7 +293,7 @@ class S3DestinationWriter:def __init__(self, ctx: DestinationRunContext) -> None:self._ctx = ctxconfig = ctx.config or {}- self._bucket: str = (config.get("bucket") or config.get("bucket_name") or "").strip("/")+ self._bucket: str = (config.get("bucket") or config.get("bucket_name") or "").strip().strip("/")self._region: str = (config.get("region") or "").strip()self._prefix: str = (config.get("prefix") or "").strip("/")self._use_virtual_style_addressing = bool(config.get("use_virtual_style_addressing"))