:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): hide raw s3 errors when linking a self-managed csv table (#118296)
Tom Owers6b2eb4136799
diff --git a/products/data_warehouse/backend/models/test/test_table.py b/products/data_warehouse/backend/models/test/test_table.pyindex d8d40fd92617..3ccb7e89a196 100644--- a/products/data_warehouse/backend/models/test/test_table.py+++ b/products/data_warehouse/backend/models/test/test_table.py@@ -982,7 +982,7 @@ class TestTable(BaseTest):with patch("products.warehouse_sources.backend.models.table.sync_execute", side_effect=side_effect):assert table.detect_csv_double_quotes_setting() is expected- def test_validate_csv_double_quotes_reraises_a_non_parse_error(self):+ def test_validate_csv_double_quotes_hides_the_raw_error_for_a_non_parse_failure(self):from clickhouse_driver.errors import ServerExceptioncredential = DataWarehouseCredential.objects.create(access_key="key", access_secret="secret", team=self.team)@@ -997,11 +997,19 @@ class TestTable(BaseTest):with patch("products.warehouse_sources.backend.models.table.sync_execute",- side_effect=ServerException("Access Denied", code=499),+ side_effect=ServerException(+ "S3 error occurred. (DB::Exception: Failed to get object info: No response body.. "+ "HTTP response code: 403: while reading 'exports/test.csv')",+ code=499,+ ),):- with pytest.raises(ServerException):+ with pytest.raises(Exception) as exc_info:table._validate_csv_double_quotes_setting()+ assert not isinstance(exc_info.value, ServerException)+ assert "s3:GetObject" in str(exc_info.value)+ assert "exports/test.csv" not in str(exc_info.value)+@parameterized.expand([("empty_folder_code_636_silent", 636, False),diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex 9ae5f09a7b44..73eef75c43bc 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -91,6 +91,8 @@ SERIALIZED_FIELD_TO_CLICKHOUSE_MAPPING: dict[DatabaseSerializedFieldType, str] =ExtractErrors = {"The AWS Access Key Id you provided does not exist": "The Access Key you provided does not exist","Access Denied: while reading key:": "Access was denied when reading a file from the bucket. Check that the provided credentials can read objects in this bucket (s3:GetObject), then try again.",+ # A HEAD request has no response body, so S3's denial reaches ClickHouse without "Access Denied".+ "Failed to get object info: No response body.. HTTP response code: 403": "Access was denied when reading a file from the bucket. Check that the provided credentials can read objects in this bucket (s3:GetObject), then try again.",# DeltaLake-kernel object_store errors (Delta-format tables, e.g. all warehouse_sources synced# tables) use a different vocabulary than ClickHouse's native S3 errors above."The operation lacked the necessary privileges to complete": "Access was denied when reading the provided file",@@ -1263,7 +1265,7 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletdef _csv_parses_with_double_quotes(self, allow_double_quotes: bool) -> bool:"""Read a few rows under one quote setting. False when the rows don't parse; any other- failure (credentials, a missing file) is raised for the caller to surface as-is."""+ failure (credentials, a missing file) is raised with a user-facing message."""ctx = HogQLContext(team_id=self.team.pk)func = build_function_call(url=self.url_pattern,@@ -1288,7 +1290,9 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletself._safe_expose_ch_error(e)if e.code in self._CSV_PARSE_ERROR_CODES:return False- raise+ if not is_expected_warehouse_read_error(e):+ capture_exception(e)+ self._safe_expose_ch_error(e)return Truedef detect_csv_double_quotes_setting(self) -> bool | None:diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.pyindex 358e1516ba33..b2b961dea4cf 100644--- a/products/warehouse_sources/backend/tests/test_table.py+++ b/products/warehouse_sources/backend/tests/test_table.py@@ -176,6 +176,11 @@ class TestSafeExposeChError:[("DB::Exception: Access Denied: while reading key: my-data/file.csv, S3 exception", "s3:GetObject"),("DB::Exception: Could not list objects in bucket my-bucket, S3 exception", "s3:ListBucket"),+ (+ "S3 error occurred. (DB::Exception: Failed to get object info: No response body.. "+ "HTTP response code: 403: while reading 'my-data/file.csv')",+ "s3:GetObject",+ ),],)def test_native_s3_access_denials_name_the_iam_action_to_check(