:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): map legacy BigQuery type names in ALTER TABLE DDL (#125192)
Tom Owersa67c38a930b4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.pyindex 13d5066d9524..692d8f22184b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py@@ -39,6 +39,14 @@ _ADD_COLUMN_CLAUSE = re.compile(r"ADD COLUMN IF NOT EXISTS\s+`([^`]+)`\s+(\w+)",# What schema autodetect gives a parquet column, in the legacy names `SchemaField` reports._AUTODETECTED_TYPE = {pa.int64(): "INTEGER", pa.float64(): "FLOAT", pa.bool_(): "BOOLEAN"}+# `get_table` reports a column's type using the legacy names regardless of which standard SQL+# name the DDL that created it used, so a column added as INT64 still comes back as INTEGER.+_REPORTED_TYPE = {"INT64": "INTEGER", "FLOAT64": "FLOAT", "BOOL": "BOOLEAN"}++# INTEGER and BOOLEAN are valid GoogleSQL DDL aliases for INT64 and BOOL, but FLOAT64 has no+# "FLOAT" alias, so real BigQuery rejects an `ADD COLUMN` naming it with "Type not found: FLOAT".+_DDL_TYPES_WITHOUT_ALIAS = {"FLOAT"}+class MergeMultiMatchError(RuntimeError):"""BigQuery's "UPDATE/MERGE must match at most one source row for each target row"."""@@ -168,7 +176,12 @@ class FakeBigQueryClient:if not match:return Falsetable = self.get_table(match.group(1))- table.add_columns(dict(_ADD_COLUMN_CLAUSE.findall(match.group(2))))+ additions: dict[str, str] = {}+ for name, field_type in _ADD_COLUMN_CLAUSE.findall(match.group(2)):+ if field_type in _DDL_TYPES_WITHOUT_ALIAS:+ raise BadRequest(f"Type not found: {field_type}")+ additions[name] = _REPORTED_TYPE.get(field_type, field_type)+ table.add_columns(additions)return Truedef _merge(self, statement: str) -> bool:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.pyindex d68c6b8bf043..34765baaabf2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py@@ -6,9 +6,10 @@ from unittest.mock import MagicMockimport pyarrow as paimport pyarrow.parquet as pq-from google.api_core.exceptions import Forbidden, NotFound, ServiceUnavailable+from google.api_core.exceptions import BadRequest, Forbidden, NotFound, ServiceUnavailablefrom google.auth.exceptions import RefreshErrorfrom google.cloud import bigquery+from parameterized import parameterizedfrom products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (DestinationBatchContext,@@ -135,8 +136,18 @@ class TestSchemaLabel:class TestDdlType:- def test_a_scalar_field_renders_its_bare_type(self) -> None:- assert _ddl_type(bigquery.SchemaField("clicks", "INTEGER")) == "INTEGER"+ @parameterized.expand(+ [+ ("STRING", "STRING"),+ # The legacy names `get_table` reports have no DDL alias for FLOAT, unlike INTEGER+ # (INT64) and BOOLEAN (BOOL) — an unmapped FLOAT fails the ALTER TABLE outright.+ ("INTEGER", "INT64"),+ ("FLOAT", "FLOAT64"),+ ("BOOLEAN", "BOOL"),+ ]+ )+ def test_a_scalar_field_renders_its_standard_sql_type(self, field_type: str, expected: str) -> None:+ assert _ddl_type(bigquery.SchemaField("clicks", field_type)) == expecteddef test_a_repeated_field_renders_as_an_array(self) -> None:# Dropping `mode` here would add the element type as a scalar column, which is@@ -153,7 +164,7 @@ class TestDdlType:fields=[bigquery.SchemaField("city", "STRING"), bigquery.SchemaField("zip", "INTEGER")],)- assert _ddl_type(field) == "STRUCT<`city` STRING, `zip` INTEGER>"+ assert _ddl_type(field) == "STRUCT<`city` STRING, `zip` INT64>"def test_a_repeated_record_field_renders_as_an_array_of_structs(self) -> None:field = bigquery.SchemaField(@@ -411,7 +422,18 @@ class TestIncremental:assert table.types["delivered_at"] == "INTEGER"assert sorted((row["id"], row["delivered_at"]) for row in table.rows) == [(1, 1700000500), (2, None)]- async def test_a_column_a_later_batch_grows_is_added_to_the_target_before_merging(self) -> None:+ @parameterized.expand(+ [+ ("clicks", 10),+ # Regression: FLOAT64 has no "FLOAT" DDL alias, so adding a float column here used+ # to fail the ALTER TABLE with "Type not found: FLOAT" before `_evolve_table`+ # normalized the legacy type name `get_table` reports.+ ("rate", 0.5),+ ]+ )+ async def test_a_column_a_later_batch_grows_is_added_to_the_target_before_merging(+ self, column: str, value: float+ ) -> None:# MERGE does not evolve schema on its own, so a column only the second batch carries# must be added to the target first, or the MERGE fails referencing a column T lacks.client = FakeBigQueryClient()@@ -422,18 +444,31 @@ class TestIncremental:await _deliver(writer,ctx,- pa.RecordBatch.from_pydict({"id": [2], "name": ["b"], "clicks": [10]}),+ pa.RecordBatch.from_pydict({"id": [2], "name": ["b"], column: [value]}),1,is_final=True,)table = client.get_table("proj.dataset.charges")- assert table.columns == ["id", "name", "clicks"]- assert sorted((row["id"], row["name"], row.get("clicks")) for row in table.rows) == [+ assert table.columns == ["id", "name", column]+ assert sorted((row["id"], row["name"], row.get(column)) for row in table.rows) == [(1, "a", None),- (2, "b", 10),+ (2, "b", value),]+ def test_the_fake_rejects_a_ddl_type_with_no_alias_the_way_bigquery_does(self) -> None:+ # Guards the float case above: if the fake accepted FLOAT in an ALTER TABLE, that case+ # would pass with or without `_ddl_type` normalizing it to FLOAT64 first.+ client = FakeBigQueryClient()+ client.load_table_from_file(+ _parquet(_rows([1], ["a"])),+ "proj.dataset.charges",+ bigquery.LoadJobConfig(write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE),+ )++ with pytest.raises(BadRequest, match="FLOAT"):+ client.query("ALTER TABLE `proj.dataset.charges` ADD COLUMN IF NOT EXISTS `rate` FLOAT")+async def test_text_that_cannot_convert_to_the_int64_column_is_a_configuration_error(self) -> None:# A retry sends the same values, so a retryable error would fail until someone changes the table.client = FakeBigQueryClient()diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.pyindex b08c2a68e8f0..03538eece8da 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py@@ -119,7 +119,9 @@ def _ddl_type(field: bigquery.SchemaField) -> str:base = (f"STRUCT<{', '.join(f'{_backtick(sub.name)} {_ddl_type(sub)}' for sub in field.fields)}>"if field.field_type in ("RECORD", "STRUCT")- else field.field_type+ # DDL only accepts the standard SQL type names: FLOAT has no DDL alias, unlike INTEGER+ # (INT64) and BOOLEAN (BOOL), so an unmapped FLOAT fails the ALTER TABLE outright.+ else _STANDARD_BIGQUERY_TYPE.get(field.field_type, field.field_type))return f"ARRAY<{base}>" if field.mode == "REPEATED" else base