:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): return a message from object storage credential probes (#120963)
Tom Owersb39074560f2f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.pyindex 01a2d1e9c67d..d89d330eb23a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py@@ -4,6 +4,7 @@ from datetime import UTC, datetimefrom typing import Generic, NoReturn, Optionalimport pyarrow as pa+from botocore.exceptions import ClientErrorfrom structlog.types import FilteringBoundLoggerfrom products.warehouse_sources.backend.models.util import validate_warehouse_table_url_pattern@@ -60,6 +61,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.schSourceSchema,incremental_field,)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_messagefrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponsefrom products.warehouse_sources.backend.types import IncrementalFieldType@@ -71,6 +73,18 @@ def _raise_translated(error: Exception, location: str) -> NoReturn:raise translated from error+def _http_status(error: BaseException) -> int | None:+ """The HTTP status of the botocore error behind an s3fs error, if there is one."""+ current: BaseException | None = error+ while current is not None:+ if isinstance(current, ClientError):+ status = current.response.get("ResponseMetadata", {}).get("HTTPStatusCode")+ if isinstance(status, int):+ return status+ current = current.__cause__+ return None++def validate_store_location(config: StoreConfig) -> None:"""Apply the checks a self-managed warehouse table URL gets: https, no internal hosts, and nobucket that PostHog itself owns."""@@ -146,6 +160,15 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDetables = self._discover(self.open_store(store_config), config)except ObjectStorageError as e:return False, str(e)+ except OSError as e:+ # s3fs turns an S3 answer without a known error code, such as a bare 401 or a 500, into a plain OSError.+ status = _http_status(e)+ if status in (401, 403):+ return False, str(translate_storage_error(PermissionError(), store_config.display_location))+ message = probe_status_message(self.get_source_config.label or "The storage provider", status)+ if message is None:+ raise+ return False, messageif not tables:return False, (f"{NO_IMPORTABLE_FILES_ERROR} in '{store_config.display_location}'. PostHog can read the bucket but "diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.pyindex 3a08922146c0..5c50e2c7a273 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py@@ -467,10 +467,7 @@ OK_NEEDS_VENDOR_BODY: set[str] = {# Sources whose probe raises for these answers. Each entry is a defect to fix: the probe must return# `(False, message)`. The value lists the answers that raise.-RAISES: dict[str, str] = {- "AmazonS3": "401 500",- "GoogleCloudStorage": "401 500",-}+RAISES: dict[str, str] = {}# Sources whose probe returns no message for these answers. Each entry is a defect to fix: the person# sees a generic message that does not say what is wrong.