Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): name the gcs role to grant when a bucket denies access (#130607)

Tom Owers435ba888bb92
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index 554002071627..5a8fc3ef6e9c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -124,6 +124,10 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
# The service whose self-managed warehouse tables this source can take over.
self_managed_service: ClassVar[StorageService]
+ # Replaces the generic access-denied text, in the wizard and on a failed sync, for a provider
+ # where the user fixes the denial somewhere specific.
+ access_denied_message: ClassVar[str | None] = None
+
@abstractmethod
def self_managed_connection_inputs(
self, location: SelfManagedLocation, credential: SelfManagedCredential
@@ -162,7 +166,8 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
return {
INVALID_CONFIG_ERROR: None,
BLOCKED_LOCATION_ERROR: None,
- ACCESS_DENIED_ERROR: (
+ ACCESS_DENIED_ERROR: self.access_denied_message
+ or (
"The storage provider rejected these credentials. Check the access key and secret, and that "
"the key can list and read the bucket, then reconnect the source."
),
@@ -199,6 +204,11 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
logger=logger,
)
+ def _validation_message(self, message: str) -> str:
+ if self.access_denied_message and message.startswith(ACCESS_DENIED_ERROR):
+ return self.access_denied_message
+ return message
+
def validate_credentials(
self,
config: ConfigType,
@@ -210,12 +220,13 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
store_config = self.object_store_config(config)
tables = self._discover(self.open_store(store_config), config)
except ObjectStorageError as e:
- return False, str(e)
+ return False, self._validation_message(str(e))
except OSError as e:
# s3fs turns an S3 answer without a known error code, such as a bare 401 or a 500, into a plain OSError.
status = _http_status(e)
if status in (401, 403):
- return False, str(translate_storage_error(PermissionError(), store_config.display_location))
+ denied = translate_storage_error(PermissionError(), store_config.display_location)
+ return False, self._validation_message(str(denied))
message = probe_status_message(self.get_source_config.label or "The storage provider", status)
if message is None:
raise
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
index 4c82a2708584..fea1280c9d4d 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py
@@ -1446,6 +1446,73 @@ def _source_with_store(store: ObjectStore) -> FakeObjectStorageSource:
return source
+def _forbidden_os_error() -> OSError:
+ error = OSError("Forbidden")
+ error.__cause__ = ClientError(
+ cast(Any, {"Error": {"Code": "403", "Message": "Forbidden"}, "ResponseMetadata": {"HTTPStatusCode": 403}}),
+ "ListObjectsV2",
+ )
+ return error
+
+
+class TestAccessDeniedMessage:
+ GENERIC_WIZARD_MESSAGE = (
+ f"{ACCESS_DENIED_ERROR} 'example-bucket'. Check the access key and secret, and that the key can list "
+ "and read objects in this bucket."
+ )
+ GENERIC_SYNC_MESSAGE = (
+ "The storage provider rejected these credentials. Check the access key and secret, and that "
+ "the key can list and read the bucket, then reconnect the source."
+ )
+ PROVIDER_MESSAGE = "This provider denied access. Grant the role on the bucket, then reconnect."
+
+ @pytest.mark.parametrize(
+ "override, wizard_message, sync_message",
+ [
+ pytest.param(None, GENERIC_WIZARD_MESSAGE, GENERIC_SYNC_MESSAGE, id="default"),
+ pytest.param(PROVIDER_MESSAGE, PROVIDER_MESSAGE, PROVIDER_MESSAGE, id="provider_override"),
+ ],
+ )
+ @pytest.mark.parametrize(
+ "denial",
+ [
+ pytest.param(PermissionError("Access Denied"), id="permission_error"),
+ pytest.param(_forbidden_os_error(), id="bare_403"),
+ ],
+ )
+ def test_a_provider_can_replace_the_access_denied_text(
+ self,
+ denial: Exception,
+ override: str | None,
+ wizard_message: str,
+ sync_message: str,
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ store = MagicMock()
+ store.list_objects.side_effect = denial
+ source = _source_with_store(store)
+ monkeypatch.setattr(FakeObjectStorageSource, "access_denied_message", override)
+
+ valid, message = source.validate_credentials(BUCKET_ROOT_CONFIG, 1)
+
+ assert not valid
+ assert message == wizard_message
+ assert source.get_non_retryable_errors()[ACCESS_DENIED_ERROR] == sync_message
+
+ def test_other_errors_keep_their_text_when_a_provider_overrides_the_access_denied_text(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ store = MagicMock()
+ store.list_objects.side_effect = FileNotFoundError("The specified bucket does not exist")
+ source = _source_with_store(store)
+ monkeypatch.setattr(FakeObjectStorageSource, "access_denied_message", self.PROVIDER_MESSAGE)
+
+ valid, message = source.validate_credentials(BUCKET_ROOT_CONFIG, 1)
+
+ assert not valid
+ assert message is not None and message.startswith(NO_SUCH_BUCKET_ERROR)
+
+
BUCKET_ROOT_CONFIG = FakeObjectStorageConfig(
values={
"url_pattern": "https://s3.us-east-1.amazonaws.com/example-bucket/",
diff --git a/products/warehouse_sources/sources/google_cloud_storage/source.py b/products/warehouse_sources/sources/google_cloud_storage/source.py
index ef0bc073b27b..d8c535062413 100644
--- a/products/warehouse_sources/sources/google_cloud_storage/source.py
+++ b/products/warehouse_sources/sources/google_cloud_storage/source.py
@@ -38,6 +38,10 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfi
api_docs_url = "https://cloud.google.com/storage/docs/interoperability"
self_managed_service = "gcs"
+ access_denied_message = (
+ "Google Cloud Storage denied access to this bucket. Check the HMAC access key and secret, and give "
+ "the service account that owns the key the Storage Object Viewer role on the bucket."
+ )
@property
def source_type(self) -> ExternalDataSourceType:
diff --git a/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.py b/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.py
index a7f37c830e26..3197e47693a2 100644
--- a/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.py
+++ b/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.py
@@ -1,10 +1,12 @@
from typing import Any
import pytest
+from unittest.mock import MagicMock
from products.warehouse_sources.sources.google_cloud_storage.source import GoogleCloudStorageSource
from products.warehouse_sources.sources.sdk import SelfManagedCredential, SourceFieldSelectConfig
from products.warehouse_sources.sources.sdk.testing import (
+ ACCESS_DENIED_ERROR,
INVALID_CONFIG_ERROR,
open_store,
parse_self_managed_url,
@@ -103,3 +105,19 @@ def test_a_self_managed_table_maps_to_its_bucket_and_credential() -> None:
assert job_inputs["url_pattern"] == "https://storage.googleapis.com/exports/"
assert (job_inputs["access_key_id"], job_inputs["secret_access_key"]) == ("AKIAEXAMPLE", "c2VjcmV0")
+
+
+def test_access_denied_names_the_role_to_grant(resolvable_hosts: None, monkeypatch: pytest.MonkeyPatch) -> None:
+ source = GoogleCloudStorageSource()
+ store = MagicMock()
+ store.list_objects.side_effect = PermissionError("Access Denied")
+ monkeypatch.setattr(source, "open_store", lambda _config: store)
+
+ valid, message = source.validate_credentials(source.parse_config(VALUES), 1)
+
+ assert not valid
+ assert message is not None
+ assert "Storage Object Viewer" in message
+ assert "access key and secret" in message
+ assert "customer_data" not in message
+ assert source.get_non_retryable_errors()[ACCESS_DENIED_ERROR] == message