Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): classify SignatureDoesNotMatch as permission denied (#117519)

Tom Owers261b7d8c155f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
index 8e30fa5c207e..276152a19a66 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/ops.py
@@ -5,6 +5,7 @@ from typing import TypedDict, TypeVar
from django.conf import settings
import deltalake
+import botocore.exceptions
import deltalake.exceptions
from structlog.types import FilteringBoundLogger
@@ -44,6 +45,14 @@ OBJECT_STORE_PERMISSION_DENIED_ERRORS = (
"The AWS Access Key Id you provided does not exist in our records.",
)
+# Codes s3fs also maps onto PermissionError (see ERROR_CODE_TO_EXCEPTION in its errors module) but
+# whose s3fs-translated message isn't the fixed, matchable text the needles above rely on — our
+# backend returns a generic "Invalid argument." Message for SignatureDoesNotMatch instead of AWS's
+# usual signature-mismatch sentence. The code itself still names a definitive, non-racy refusal
+# (unlike a bodyless 403, see is_object_store_permission_denied), so it's read off the chained
+# ClientError instead of guessed from message text.
+_PERMISSION_DENIED_CLIENT_ERROR_CODES = ("SignatureDoesNotMatch",)
+
# Reaches the customer as the sync run's error text, so it names neither the bucket nor the object
# key. The data warehouse bucket is PostHog's own and the worker authenticates to it with its own
# role, which means a refusal here is never caused by the customer's source or credentials.
@@ -71,6 +80,13 @@ class ObjectStorePermissionDeniedError(Exception):
def is_object_store_permission_denied(error: BaseException) -> bool:
+ cause = error.__cause__
+ if (
+ isinstance(error, OSError)
+ and isinstance(cause, botocore.exceptions.ClientError)
+ and cause.response.get("Error", {}).get("Code") in _PERMISSION_DENIED_CLIENT_ERROR_CODES
+ ):
+ return True
return isinstance(error, OSError | deltalake.exceptions.DeltaError) and any(
needle in str(error) for needle in OBJECT_STORE_PERMISSION_DENIED_ERRORS
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
index 6254811eb84d..e5a350a1a3dd 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_ops.py
@@ -4,6 +4,7 @@ from unittest.mock import MagicMock
from django.test import override_settings
import deltalake
+import botocore.exceptions
from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
@@ -18,6 +19,18 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.del
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.test.helpers import make_logger
+def _permission_error_with_client_error_cause() -> PermissionError:
+ # Mirrors s3fs.errors.translate_boto_error: the message on the PermissionError it raises is the
+ # server's own (here generic) Message, while the original ClientError (which carries the real
+ # Code) is kept only as __cause__.
+ client_error = botocore.exceptions.ClientError(
+ {"Error": {"Code": "SignatureDoesNotMatch", "Message": "Invalid argument."}}, "PutObject"
+ )
+ permission_error = PermissionError("Invalid argument.")
+ permission_error.__cause__ = client_error
+ return permission_error
+
+
class TestDeltaMergeSpillKwargs:
# Guards the wiring from settings → delta-rs merge kwargs. A silent break here (renamed setting,
# dropped forwarding, or emitting a None kwarg) stops merges spilling to disk and OOMs return.
@@ -205,6 +218,12 @@ class TestExecuteWithConflictRetry:
"s3fs_invalid_access_key_id",
PermissionError("The AWS Access Key Id you provided does not exist in our records."),
),
+ # s3fs also collapses SignatureDoesNotMatch into PermissionError, but our backend returns
+ # a generic "Invalid argument." Message for it instead of AWS's usual signature-mismatch
+ # sentence, so none of the message needles above match it. Regression coverage for this
+ # refusal leaking as a raw, unclassified PermissionError instead of being read off the
+ # chained ClientError's actual code.
+ ("s3fs_signature_does_not_match", _permission_error_with_client_error_cause()),
]
)
@pytest.mark.asyncio