Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): classify bucket connection blips as transient in delta table opens (#126728)

Tom Owersf0cb2b3e1794
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py
index ccbeaf0948b5..dd80631d88f7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/table.py
@@ -458,7 +458,14 @@ class DeltaTableRef:
async def _open_delta_table(self) -> deltalake.DeltaTable | None:
delta_uri = await self._get_delta_table_uri()
- storage_options = self._get_credentials()
+ try:
+ storage_options = self._get_credentials()
+ except Exception as e:
+ # Unlike the open attempts below, this call isn't delta-rs's own object_store client, so
+ # an unreachable/bootstrapping bucket (ensure_bucket_exists's EndpointConnectionError under
+ # USE_LOCAL_SETUP) would otherwise escape uncaught here and reach the caller unclassified.
+ await self._capture_unless_transient(e)
+ raise
table = await self._open_directly(delta_uri, storage_options)
if table is not None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py
index eeee595a48f3..ff78beb5d359 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/delta/test/test_table.py
@@ -13,6 +13,7 @@ import pyarrow as pa
import deltalake
import deltalite
import pyarrow.parquet as pq
+import botocore.exceptions
from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.deltalite_handles import (
@@ -739,6 +740,32 @@ class TestGetDeltaTableUnrecoverableErrors:
mock_purge.assert_not_awaited()
assert table_ref.is_first_sync is False
+ @pytest.mark.asyncio
+ async def test_get_credentials_transient_error_is_not_captured_but_still_reraised(self):
+ """`_get_credentials` (ensure_bucket_exists under USE_LOCAL_SETUP) runs before any open
+ attempt and isn't delta-rs's own object_store client, so its errors previously escaped
+ `_capture_unless_transient` entirely: a self-hosted object store still bootstrapping (raising
+ botocore's EndpointConnectionError) was captured as a defect on every occurrence instead of
+ being recognized as the same transient blip `is_transient_object_store_error` already covers."""
+ table_ref = DeltaTableRef(resource_name="t", job=MagicMock(), logger=make_logger())
+ delta_uri = "s3://bucket/team_id/job_id/t"
+
+ original_error = botocore.exceptions.EndpointConnectionError(endpoint_url="http://objectstorage:19000")
+ module = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table"
+ with (
+ patch.object(table_ref, "_get_delta_table_uri", AsyncMock(return_value=delta_uri)),
+ patch.object(table_ref, "_get_credentials", MagicMock(side_effect=original_error)),
+ patch(f"{module}.deltalake.DeltaTable") as mock_delta_table,
+ patch(f"{module}.capture_exception") as mock_capture,
+ ):
+ with pytest.raises(TransientObjectStoreError) as exc_info:
+ await table_ref.get_delta_table()
+
+ assert exc_info.value.__cause__ is original_error
+ mock_capture.assert_not_called()
+ mock_delta_table.assert_not_called()
+ mock_delta_table.is_deltatable.assert_not_called()
+
class TestIsTableCorrupted:
_MODULE = "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table"