Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): return a message from object storage credential probes (#120963)

Tom Owersb39074560f2f
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
index 01a2d1e9c67d..d89d330eb23a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/common/object_store/source_base.py
@@ -4,6 +4,7 @@ from datetime import UTC, datetime
from typing import Generic, NoReturn, Optional
import pyarrow as pa
+from botocore.exceptions import ClientError
from structlog.types import FilteringBoundLogger
from products.warehouse_sources.backend.models.util import validate_warehouse_table_url_pattern
@@ -60,6 +61,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.sch
SourceSchema,
incremental_field,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.source_helpers import probe_status_message
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse
from products.warehouse_sources.backend.types import IncrementalFieldType
@@ -71,6 +73,18 @@ def _raise_translated(error: Exception, location: str) -> NoReturn:
raise translated from error
+def _http_status(error: BaseException) -> int | None:
+ """The HTTP status of the botocore error behind an s3fs error, if there is one."""
+ current: BaseException | None = error
+ while current is not None:
+ if isinstance(current, ClientError):
+ status = current.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
+ if isinstance(status, int):
+ return status
+ current = current.__cause__
+ return None
+
+
def validate_store_location(config: StoreConfig) -> None:
"""Apply the checks a self-managed warehouse table URL gets: https, no internal hosts, and no
bucket that PostHog itself owns."""
@@ -146,6 +160,15 @@ class ObjectStorageSource(SimpleSource[ConfigType], CursorSource[ObjectStorageDe
tables = self._discover(self.open_store(store_config), config)
except ObjectStorageError as e:
return False, str(e)
+ except OSError as e:
+ # s3fs turns an S3 answer without a known error code, such as a bare 401 or a 500, into a plain OSError.
+ status = _http_status(e)
+ if status in (401, 403):
+ return False, str(translate_storage_error(PermissionError(), store_config.display_location))
+ message = probe_status_message(self.get_source_config.label or "The storage provider", status)
+ if message is None:
+ raise
+ return False, message
if not tables:
return False, (
f"{NO_IMPORTABLE_FILES_ERROR} in '{store_config.display_location}'. PostHog can read the bucket but "
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
index 3a08922146c0..5c50e2c7a273 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/tests/test_source_credential_contract.py
@@ -467,10 +467,7 @@ OK_NEEDS_VENDOR_BODY: set[str] = {
# Sources whose probe raises for these answers. Each entry is a defect to fix: the probe must return
# `(False, message)`. The value lists the answers that raise.
-RAISES: dict[str, str] = {
- "AmazonS3": "401 500",
- "GoogleCloudStorage": "401 500",
-}
+RAISES: dict[str, str] = {}
# Sources whose probe returns no message for these answers. Each entry is a defect to fix: the person
# sees a generic message that does not say what is wrong.