:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): name the gcs role to grant when a bucket denies access (#130607)
Tom Owers435ba888bb92
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pyindex 554002071627..5a8fc3ef6e9c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -124,6 +124,10 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe# The service whose self-managed warehouse tables this source can take over.self_managed_service: ClassVar[StorageService]+ # Replaces the generic access-denied text, in the wizard and on a failed sync, for a provider+ # where the user fixes the denial somewhere specific.+ access_denied_message: ClassVar[str | None] = None+@abstractmethoddef self_managed_connection_inputs(self, location: SelfManagedLocation, credential: SelfManagedCredential@@ -162,7 +166,8 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDereturn {INVALID_CONFIG_ERROR: None,BLOCKED_LOCATION_ERROR: None,- ACCESS_DENIED_ERROR: (+ ACCESS_DENIED_ERROR: self.access_denied_message+ or ("The storage provider rejected these credentials. Check the access key and secret, and that ""the key can list and read the bucket, then reconnect the source."),@@ -199,6 +204,11 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDelogger=logger,)+ def _validation_message(self, message: str) -> str:+ if self.access_denied_message and message.startswith(ACCESS_DENIED_ERROR):+ return self.access_denied_message+ return message+def validate_credentials(self,config: ConfigType,@@ -210,12 +220,13 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDestore_config = self.object_store_config(config)tables = self._discover(self.open_store(store_config), config)except ObjectStorageError as e:- return False, str(e)+ return False, self._validation_message(str(e))except OSError as e:# s3fs turns an S3 answer without a known error code, such as a bare 401 or a 500, into a plain OSError.status = _http_status(e)if status in (401, 403):- return False, str(translate_storage_error(PermissionError(), store_config.display_location))+ denied = translate_storage_error(PermissionError(), store_config.display_location)+ return False, self._validation_message(str(denied))message = probe_status_message(self.get_source_config.label or "The storage provider", status)if message is None:raisediff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.pyindex 4c82a2708584..fea1280c9d4d 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/tests/test_object_store.py@@ -1446,6 +1446,73 @@ def _source_with_store(store: ObjectStore) -> FakeObjectStorageSource:return source+def _forbidden_os_error() -> OSError:+ error = OSError("Forbidden")+ error.__cause__ = ClientError(+ cast(Any, {"Error": {"Code": "403", "Message": "Forbidden"}, "ResponseMetadata": {"HTTPStatusCode": 403}}),+ "ListObjectsV2",+ )+ return error+++class TestAccessDeniedMessage:+ GENERIC_WIZARD_MESSAGE = (+ f"{ACCESS_DENIED_ERROR} 'example-bucket'. Check the access key and secret, and that the key can list "+ "and read objects in this bucket."+ )+ GENERIC_SYNC_MESSAGE = (+ "The storage provider rejected these credentials. Check the access key and secret, and that "+ "the key can list and read the bucket, then reconnect the source."+ )+ PROVIDER_MESSAGE = "This provider denied access. Grant the role on the bucket, then reconnect."++ @pytest.mark.parametrize(+ "override, wizard_message, sync_message",+ [+ pytest.param(None, GENERIC_WIZARD_MESSAGE, GENERIC_SYNC_MESSAGE, id="default"),+ pytest.param(PROVIDER_MESSAGE, PROVIDER_MESSAGE, PROVIDER_MESSAGE, id="provider_override"),+ ],+ )+ @pytest.mark.parametrize(+ "denial",+ [+ pytest.param(PermissionError("Access Denied"), id="permission_error"),+ pytest.param(_forbidden_os_error(), id="bare_403"),+ ],+ )+ def test_a_provider_can_replace_the_access_denied_text(+ self,+ denial: Exception,+ override: str | None,+ wizard_message: str,+ sync_message: str,+ monkeypatch: pytest.MonkeyPatch,+ ) -> None:+ store = MagicMock()+ store.list_objects.side_effect = denial+ source = _source_with_store(store)+ monkeypatch.setattr(FakeObjectStorageSource, "access_denied_message", override)++ valid, message = source.validate_credentials(BUCKET_ROOT_CONFIG, 1)++ assert not valid+ assert message == wizard_message+ assert source.get_non_retryable_errors()[ACCESS_DENIED_ERROR] == sync_message++ def test_other_errors_keep_their_text_when_a_provider_overrides_the_access_denied_text(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ store = MagicMock()+ store.list_objects.side_effect = FileNotFoundError("The specified bucket does not exist")+ source = _source_with_store(store)+ monkeypatch.setattr(FakeObjectStorageSource, "access_denied_message", self.PROVIDER_MESSAGE)++ valid, message = source.validate_credentials(BUCKET_ROOT_CONFIG, 1)++ assert not valid+ assert message is not None and message.startswith(NO_SUCH_BUCKET_ERROR)++BUCKET_ROOT_CONFIG = FakeObjectStorageConfig(values={"url_pattern": "https://s3.us-east-1.amazonaws.com/example-bucket/",diff --git a/products/warehouse_sources/sources/google_cloud_storage/source.py b/products/warehouse_sources/sources/google_cloud_storage/source.pyindex ef0bc073b27b..d8c535062413 100644--- a/products/warehouse_sources/sources/google_cloud_storage/source.py+++ b/products/warehouse_sources/sources/google_cloud_storage/source.py@@ -38,6 +38,10 @@ class GoogleCloudStorageSource(ObjectStorageSource[GoogleCloudStorageSourceConfiapi_docs_url = "https://cloud.google.com/storage/docs/interoperability"self_managed_service = "gcs"+ access_denied_message = (+ "Google Cloud Storage denied access to this bucket. Check the HMAC access key and secret, and give "+ "the service account that owns the key the Storage Object Viewer role on the bucket."+ )@propertydef source_type(self) -> ExternalDataSourceType:diff --git a/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.py b/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.pyindex a7f37c830e26..3197e47693a2 100644--- a/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.py+++ b/products/warehouse_sources/sources/google_cloud_storage/tests/test_google_cloud_storage_store_config.py@@ -1,10 +1,12 @@from typing import Anyimport pytest+from unittest.mock import MagicMockfrom products.warehouse_sources.sources.google_cloud_storage.source import GoogleCloudStorageSourcefrom products.warehouse_sources.sources.sdk import SelfManagedCredential, SourceFieldSelectConfigfrom products.warehouse_sources.sources.sdk.testing import (+ ACCESS_DENIED_ERROR,INVALID_CONFIG_ERROR,open_store,parse_self_managed_url,@@ -103,3 +105,19 @@ def test_a_self_managed_table_maps_to_its_bucket_and_credential() -> None:assert job_inputs["url_pattern"] == "https://storage.googleapis.com/exports/"assert (job_inputs["access_key_id"], job_inputs["secret_access_key"]) == ("AKIAEXAMPLE", "c2VjcmV0")+++def test_access_denied_names_the_role_to_grant(resolvable_hosts: None, monkeypatch: pytest.MonkeyPatch) -> None:+ source = GoogleCloudStorageSource()+ store = MagicMock()+ store.list_objects.side_effect = PermissionError("Access Denied")+ monkeypatch.setattr(source, "open_store", lambda _config: store)++ valid, message = source.validate_credentials(source.parse_config(VALUES), 1)++ assert not valid+ assert message is not None+ assert "Storage Object Viewer" in message+ assert "access key and secret" in message+ assert "customer_data" not in message+ assert source.get_non_retryable_errors()[ACCESS_DENIED_ERROR] == message