Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): map legacy BigQuery type names in ALTER TABLE DDL (#125192)

Tom Owersa67c38a930b4
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
index 13d5066d9524..692d8f22184b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
@@ -39,6 +39,14 @@ _ADD_COLUMN_CLAUSE = re.compile(r"ADD COLUMN IF NOT EXISTS\s+`([^`]+)`\s+(\w+)",
# What schema autodetect gives a parquet column, in the legacy names `SchemaField` reports.
_AUTODETECTED_TYPE = {pa.int64(): "INTEGER", pa.float64(): "FLOAT", pa.bool_(): "BOOLEAN"}
+# `get_table` reports a column's type using the legacy names regardless of which standard SQL
+# name the DDL that created it used, so a column added as INT64 still comes back as INTEGER.
+_REPORTED_TYPE = {"INT64": "INTEGER", "FLOAT64": "FLOAT", "BOOL": "BOOLEAN"}
+
+# INTEGER and BOOLEAN are valid GoogleSQL DDL aliases for INT64 and BOOL, but FLOAT64 has no
+# "FLOAT" alias, so real BigQuery rejects an `ADD COLUMN` naming it with "Type not found: FLOAT".
+_DDL_TYPES_WITHOUT_ALIAS = {"FLOAT"}
+
class MergeMultiMatchError(RuntimeError):
"""BigQuery's "UPDATE/MERGE must match at most one source row for each target row"."""
@@ -168,7 +176,12 @@ class FakeBigQueryClient:
if not match:
return False
table = self.get_table(match.group(1))
- table.add_columns(dict(_ADD_COLUMN_CLAUSE.findall(match.group(2))))
+ additions: dict[str, str] = {}
+ for name, field_type in _ADD_COLUMN_CLAUSE.findall(match.group(2)):
+ if field_type in _DDL_TYPES_WITHOUT_ALIAS:
+ raise BadRequest(f"Type not found: {field_type}")
+ additions[name] = _REPORTED_TYPE.get(field_type, field_type)
+ table.add_columns(additions)
return True
def _merge(self, statement: str) -> bool:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
index d68c6b8bf043..34765baaabf2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
@@ -6,9 +6,10 @@ from unittest.mock import MagicMock
import pyarrow as pa
import pyarrow.parquet as pq
-from google.api_core.exceptions import Forbidden, NotFound, ServiceUnavailable
+from google.api_core.exceptions import BadRequest, Forbidden, NotFound, ServiceUnavailable
from google.auth.exceptions import RefreshError
from google.cloud import bigquery
+from parameterized import parameterized
from products.warehouse_sources.backend.temporal.data_imports.destinations.contracts import (
DestinationBatchContext,
@@ -135,8 +136,18 @@ class TestSchemaLabel:
class TestDdlType:
- def test_a_scalar_field_renders_its_bare_type(self) -> None:
- assert _ddl_type(bigquery.SchemaField("clicks", "INTEGER")) == "INTEGER"
+ @parameterized.expand(
+ [
+ ("STRING", "STRING"),
+ # The legacy names `get_table` reports have no DDL alias for FLOAT, unlike INTEGER
+ # (INT64) and BOOLEAN (BOOL) — an unmapped FLOAT fails the ALTER TABLE outright.
+ ("INTEGER", "INT64"),
+ ("FLOAT", "FLOAT64"),
+ ("BOOLEAN", "BOOL"),
+ ]
+ )
+ def test_a_scalar_field_renders_its_standard_sql_type(self, field_type: str, expected: str) -> None:
+ assert _ddl_type(bigquery.SchemaField("clicks", field_type)) == expected
def test_a_repeated_field_renders_as_an_array(self) -> None:
# Dropping `mode` here would add the element type as a scalar column, which is
@@ -153,7 +164,7 @@ class TestDdlType:
fields=[bigquery.SchemaField("city", "STRING"), bigquery.SchemaField("zip", "INTEGER")],
)
- assert _ddl_type(field) == "STRUCT<`city` STRING, `zip` INTEGER>"
+ assert _ddl_type(field) == "STRUCT<`city` STRING, `zip` INT64>"
def test_a_repeated_record_field_renders_as_an_array_of_structs(self) -> None:
field = bigquery.SchemaField(
@@ -411,7 +422,18 @@ class TestIncremental:
assert table.types["delivered_at"] == "INTEGER"
assert sorted((row["id"], row["delivered_at"]) for row in table.rows) == [(1, 1700000500), (2, None)]
- async def test_a_column_a_later_batch_grows_is_added_to_the_target_before_merging(self) -> None:
+ @parameterized.expand(
+ [
+ ("clicks", 10),
+ # Regression: FLOAT64 has no "FLOAT" DDL alias, so adding a float column here used
+ # to fail the ALTER TABLE with "Type not found: FLOAT" before `_evolve_table`
+ # normalized the legacy type name `get_table` reports.
+ ("rate", 0.5),
+ ]
+ )
+ async def test_a_column_a_later_batch_grows_is_added_to_the_target_before_merging(
+ self, column: str, value: float
+ ) -> None:
# MERGE does not evolve schema on its own, so a column only the second batch carries
# must be added to the target first, or the MERGE fails referencing a column T lacks.
client = FakeBigQueryClient()
@@ -422,18 +444,31 @@ class TestIncremental:
await _deliver(
writer,
ctx,
- pa.RecordBatch.from_pydict({"id": [2], "name": ["b"], "clicks": [10]}),
+ pa.RecordBatch.from_pydict({"id": [2], "name": ["b"], column: [value]}),
1,
is_final=True,
)
table = client.get_table("proj.dataset.charges")
- assert table.columns == ["id", "name", "clicks"]
- assert sorted((row["id"], row["name"], row.get("clicks")) for row in table.rows) == [
+ assert table.columns == ["id", "name", column]
+ assert sorted((row["id"], row["name"], row.get(column)) for row in table.rows) == [
(1, "a", None),
- (2, "b", 10),
+ (2, "b", value),
]
+ def test_the_fake_rejects_a_ddl_type_with_no_alias_the_way_bigquery_does(self) -> None:
+ # Guards the float case above: if the fake accepted FLOAT in an ALTER TABLE, that case
+ # would pass with or without `_ddl_type` normalizing it to FLOAT64 first.
+ client = FakeBigQueryClient()
+ client.load_table_from_file(
+ _parquet(_rows([1], ["a"])),
+ "proj.dataset.charges",
+ bigquery.LoadJobConfig(write_disposition=bigquery.WriteDisposition.WRITE_TRUNCATE),
+ )
+
+ with pytest.raises(BadRequest, match="FLOAT"):
+ client.query("ALTER TABLE `proj.dataset.charges` ADD COLUMN IF NOT EXISTS `rate` FLOAT")
+
async def test_text_that_cannot_convert_to_the_int64_column_is_a_configuration_error(self) -> None:
# A retry sends the same values, so a retryable error would fail until someone changes the table.
client = FakeBigQueryClient()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
index b08c2a68e8f0..03538eece8da 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
@@ -119,7 +119,9 @@ def _ddl_type(field: bigquery.SchemaField) -> str:
base = (
f"STRUCT<{', '.join(f'{_backtick(sub.name)} {_ddl_type(sub)}' for sub in field.fields)}>"
if field.field_type in ("RECORD", "STRUCT")
- else field.field_type
+ # DDL only accepts the standard SQL type names: FLOAT has no DDL alias, unlike INTEGER
+ # (INT64) and BOOLEAN (BOOL), so an unmapped FLOAT fails the ALTER TABLE outright.
+ else _STANDARD_BIGQUERY_TYPE.get(field.field_type, field.field_type)
)
return f"ARRAY<{base}>" if field.mode == "REPEATED" else base