Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): evolve the BigQuery merge target before running MERGE (#123183)

Tom Owers9b44efec8354
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
index d46aa42de604..13d5066d9524 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
@@ -5,9 +5,9 @@ re-applied or a finished run is replayed, so a test that only counted calls woul
This fake keeps rows and labels per table and applies the writer's load, copy, delete, merge and
drop-column statements to them, which lets a test assert the table's contents.
-It also refuses a `MERGE` whose source matches one target row twice, or whose source holds a
-column as a different type than the target, the way BigQuery does, so those batches fail here
-too rather than passing quietly.
+It also refuses a `MERGE` whose source matches one target row twice, whose source holds a column
+the target does not have, or whose source holds a column as a different type than the target, the
+way BigQuery does, so those batches fail here too rather than passing quietly.
"""
from __future__ import annotations
@@ -32,6 +32,8 @@ _DROP_COLUMN = re.compile(
r"ALTER TABLE\s+`([^`]+)`\s+DROP COLUMN IF EXISTS\s+`([^`]+)`",
re.IGNORECASE,
)
+_ADD_COLUMN = re.compile(r"ALTER TABLE\s+`([^`]+)`\s+(ADD COLUMN IF NOT EXISTS.*)", re.IGNORECASE)
+_ADD_COLUMN_CLAUSE = re.compile(r"ADD COLUMN IF NOT EXISTS\s+`([^`]+)`\s+(\w+)", re.IGNORECASE)
# What schema autodetect gives a parquet column, in the legacy names `SchemaField` reports.
@@ -133,7 +135,7 @@ class FakeBigQueryClient:
return _Job()
def query(self, statement: str) -> _Job:
- for handler in (self._delete, self._drop_column, self._merge):
+ for handler in (self._delete, self._drop_column, self._add_column, self._merge):
if handler(statement):
return _Job()
raise AssertionError(f"FakeBigQueryClient cannot run: {statement}")
@@ -161,6 +163,14 @@ class FakeBigQueryClient:
row.pop(column, None)
return True
+ def _add_column(self, statement: str) -> bool:
+ match = _ADD_COLUMN.search(statement)
+ if not match:
+ return False
+ table = self.get_table(match.group(1))
+ table.add_columns(dict(_ADD_COLUMN_CLAUSE.findall(match.group(2))))
+ return True
+
def _merge(self, statement: str) -> bool:
tables = _MERGE_TABLES.search(statement)
if not tables:
@@ -169,12 +179,15 @@ class FakeBigQueryClient:
target = self.get_table(tables.group(1))
source = self.get_table(tables.group(2))
keys = [target_key for target_key, _ in _MERGE_KEYS.findall(tables.group(3))]
+ # Real BigQuery does not evolve the target's schema for a MERGE: a source column the
+ # target lacks fails the query outright, which is what `_evolve_table` exists to avoid.
for column, field_type in source.types.items():
- if column in target.types and target.types[column] != field_type:
+ if column not in target.types:
+ raise BadRequest(f"Name {column} not found inside T")
+ if target.types[column] != field_type:
raise BadRequest(
f"Value of type {field_type} cannot be assigned to T.{column}, which has type {target.types[column]}"
)
- target.add_columns(source.types)
incoming_per_key = Counter(_key_of(row, keys) for row in source.rows)
for row in target.rows:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
index 46aacb47d139..d68c6b8bf043 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline
BigQueryDestinationConfigurationError,
BigQueryDestinationWriter,
UnrelatedTableExistsError,
+ _ddl_type,
staging_table_name,
)
@@ -133,6 +134,35 @@ class TestSchemaLabel:
assert all(c.islower() or c.isdigit() or c in "_-" for c in label)
+class TestDdlType:
+ def test_a_scalar_field_renders_its_bare_type(self) -> None:
+ assert _ddl_type(bigquery.SchemaField("clicks", "INTEGER")) == "INTEGER"
+
+ def test_a_repeated_field_renders_as_an_array(self) -> None:
+ # Dropping `mode` here would add the element type as a scalar column, which is
+ # incompatible with the array BigQuery already autodetected for the scratch table.
+ field = bigquery.SchemaField("tags", "STRING", mode="REPEATED")
+
+ assert _ddl_type(field) == "ARRAY<STRING>"
+
+ def test_a_record_field_renders_as_a_struct_of_its_subfields(self) -> None:
+ # A bare `RECORD` is not valid DDL; the subfields must be spelled out.
+ field = bigquery.SchemaField(
+ "address",
+ "RECORD",
+ fields=[bigquery.SchemaField("city", "STRING"), bigquery.SchemaField("zip", "INTEGER")],
+ )
+
+ assert _ddl_type(field) == "STRUCT<`city` STRING, `zip` INTEGER>"
+
+ def test_a_repeated_record_field_renders_as_an_array_of_structs(self) -> None:
+ field = bigquery.SchemaField(
+ "addresses", "RECORD", mode="REPEATED", fields=[bigquery.SchemaField("city", "STRING")]
+ )
+
+ assert _ddl_type(field) == "ARRAY<STRUCT<`city` STRING>>"
+
+
class TestCheckOwnedOrAbsent:
def test_a_table_that_does_not_exist_is_reported_absent(self) -> None:
writer = BigQueryDestinationWriter(_ctx())
@@ -381,6 +411,29 @@ class TestIncremental:
assert table.types["delivered_at"] == "INTEGER"
assert sorted((row["id"], row["delivered_at"]) for row in table.rows) == [(1, 1700000500), (2, None)]
+ async def test_a_column_a_later_batch_grows_is_added_to_the_target_before_merging(self) -> None:
+ # MERGE does not evolve schema on its own, so a column only the second batch carries
+ # must be added to the target first, or the MERGE fails referencing a column T lacks.
+ client = FakeBigQueryClient()
+ ctx = _ctx(sync_type="incremental", primary_keys=("id",))
+ writer = LocalBigQueryWriter(ctx, client)
+
+ await _deliver(writer, ctx, _rows([1], ["a"]), 0)
+ await _deliver(
+ writer,
+ ctx,
+ pa.RecordBatch.from_pydict({"id": [2], "name": ["b"], "clicks": [10]}),
+ 1,
+ is_final=True,
+ )
+
+ table = client.get_table("proj.dataset.charges")
+ assert table.columns == ["id", "name", "clicks"]
+ assert sorted((row["id"], row["name"], row.get("clicks")) for row in table.rows) == [
+ (1, "a", None),
+ (2, "b", 10),
+ ]
+
async def test_text_that_cannot_convert_to_the_int64_column_is_a_configuration_error(self) -> None:
# A retry sends the same values, so a retryable error would fail until someone changes the table.
client = FakeBigQueryClient()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
index a8b54fb6af61..b08c2a68e8f0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
@@ -109,6 +109,21 @@ def _bigquery_type_of(arrow_type: pa.DataType) -> str | None:
return None
+def _ddl_type(field: bigquery.SchemaField) -> str:
+ """The DDL type declaration an `ALTER TABLE ADD COLUMN` needs for this field.
+
+ `field.field_type` alone is only the element type: a `REPEATED` field needs `ARRAY<...>`
+ around it, and a `RECORD`/`STRUCT` field needs its subfields spelled out, recursively for a
+ nested struct or a struct inside an array.
+ """
+ base = (
+ f"STRUCT<{', '.join(f'{_backtick(sub.name)} {_ddl_type(sub)}' for sub in field.fields)}>"
+ if field.field_type in ("RECORD", "STRUCT")
+ else field.field_type
+ )
+ return f"ARRAY<{base}>" if field.mode == "REPEATED" else base
+
+
class BigQueryDestinationConfigurationError(ValueError):
"""The destination's config cannot produce a valid BigQuery write."""
@@ -404,19 +419,20 @@ class BigQueryDestinationWriter:
temp = f"{run.table_name}__ph_tmp_{run_scope(run.run_uuid, 8)}_{ctx.batch_index}_{chunk}"
target_ref = self._table_ref(run.table_name)
target = self._owned_table(client, target_ref, "merge into it")
- target_absent = target is None
if target is not None:
table = self._align_to_target(table, target)
self._load(client, temp, table, bigquery.WriteDisposition.WRITE_TRUNCATE)
try:
- if target_absent:
+ if target is None:
# `MERGE` fails outright against a target that does not exist, and an
# incremental schema is on `sync_type` "incremental" from its very first run,
# so nothing else ever creates it. Take the schema BigQuery just derived for the
# scratch table rather than mapping Arrow types a second time.
self._create_like(client, target_ref, temp)
self._mark_owned(client, target_ref)
+ else:
+ self._evolve_table(client, target_ref, target.schema, temp, list(table.schema.names))
self._merge(client, run.table_name, temp, list(table.schema.names), primary_keys)
finally:
client.delete_table(self._table_ref(temp), not_found_ok=True)
@@ -451,6 +467,32 @@ class BigQueryDestinationWriter:
table = table.set_column(index, pa.field(field.name, arrow_type), cast)
return table
+ def _evolve_table(
+ self,
+ client: bigquery.Client,
+ target_ref: str,
+ target_schema: list[bigquery.SchemaField],
+ temp: str,
+ column_names: list[str],
+ ) -> None:
+ """Add columns the source chunk has grown, before `MERGE` can reference them.
+
+ Unlike `_load`'s `ALLOW_FIELD_ADDITION` on the plain-append path, `MERGE` does not
+ evolve the target's schema on its own, so a column new to this chunk needs adding to
+ the target first — otherwise the `MERGE` fails referencing a column T does not have.
+ Additive only, same as the other SQL writers' `_evolve_table`. Skips the extra
+ `get_table` call for the scratch table's schema when the chunk carries no column the
+ target lacks, which is the common case on an incremental sync's steady-state batches.
+ """
+ existing = {field.name for field in target_schema}
+ if all(name in existing for name in column_names):
+ return
+ additions = [field for field in client.get_table(self._table_ref(temp)).schema if field.name not in existing]
+ if not additions:
+ return
+ clauses = ", ".join(f"ADD COLUMN IF NOT EXISTS {_backtick(f.name)} {_ddl_type(f)}" for f in additions)
+ client.query(f"ALTER TABLE {_backtick(target_ref)} {clauses}").result()
+
def _create_like(self, client: bigquery.Client, table_ref: str, model: str) -> None:
"""Create an empty table carrying another table's schema."""
client.create_table(bigquery.Table(table_ref, schema=client.get_table(self._table_ref(model)).schema))