:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): evolve the BigQuery merge target before running MERGE (#123183)
Tom Owers9b44efec8354
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.pyindex d46aa42de604..13d5066d9524 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py@@ -5,9 +5,9 @@ re-applied or a finished run is replayed, so a test that only counted calls woulThis fake keeps rows and labels per table and applies the writer's load, copy, delete, merge anddrop-column statements to them, which lets a test assert the table's contents.-It also refuses a `MERGE` whose source matches one target row twice, or whose source holds a-column as a different type than the target, the way BigQuery does, so those batches fail here-too rather than passing quietly.+It also refuses a `MERGE` whose source matches one target row twice, whose source holds a column+the target does not have, or whose source holds a column as a different type than the target, the+way BigQuery does, so those batches fail here too rather than passing quietly."""from __future__ import annotations@@ -32,6 +32,8 @@ _DROP_COLUMN = re.compile(r"ALTER TABLE\s+`([^`]+)`\s+DROP COLUMN IF EXISTS\s+`([^`]+)`",re.IGNORECASE,)+_ADD_COLUMN = re.compile(r"ALTER TABLE\s+`([^`]+)`\s+(ADD COLUMN IF NOT EXISTS.*)", re.IGNORECASE)+_ADD_COLUMN_CLAUSE = re.compile(r"ADD COLUMN IF NOT EXISTS\s+`([^`]+)`\s+(\w+)", re.IGNORECASE)# What schema autodetect gives a parquet column, in the legacy names `SchemaField` reports.@@ -133,7 +135,7 @@ class FakeBigQueryClient:return _Job()def query(self, statement: str) -> _Job:- for handler in (self._delete, self._drop_column, self._merge):+ for handler in (self._delete, self._drop_column, self._add_column, self._merge):if handler(statement):return _Job()raise AssertionError(f"FakeBigQueryClient cannot run: {statement}")@@ -161,6 +163,14 @@ class FakeBigQueryClient:row.pop(column, None)return True+ def _add_column(self, statement: str) -> bool:+ match = _ADD_COLUMN.search(statement)+ if not match:+ return False+ table = self.get_table(match.group(1))+ table.add_columns(dict(_ADD_COLUMN_CLAUSE.findall(match.group(2))))+ return True+def _merge(self, statement: str) -> bool:tables = _MERGE_TABLES.search(statement)if not tables:@@ -169,12 +179,15 @@ class FakeBigQueryClient:target = self.get_table(tables.group(1))source = self.get_table(tables.group(2))keys = [target_key for target_key, _ in _MERGE_KEYS.findall(tables.group(3))]+ # Real BigQuery does not evolve the target's schema for a MERGE: a source column the+ # target lacks fails the query outright, which is what `_evolve_table` exists to avoid.for column, field_type in source.types.items():- if column in target.types and target.types[column] != field_type:+ if column not in target.types:+ raise BadRequest(f"Name {column} not found inside T")+ if target.types[column] != field_type:raise BadRequest(f"Value of type {field_type} cannot be assigned to T.{column}, which has type {target.types[column]}")- target.add_columns(source.types)incoming_per_key = Counter(_key_of(row, keys) for row in source.rows)for row in target.rows:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.pyindex 46aacb47d139..d68c6b8bf043 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py@@ -26,6 +26,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipelineBigQueryDestinationConfigurationError,BigQueryDestinationWriter,UnrelatedTableExistsError,+ _ddl_type,staging_table_name,)@@ -133,6 +134,35 @@ class TestSchemaLabel:assert all(c.islower() or c.isdigit() or c in "_-" for c in label)+class TestDdlType:+ def test_a_scalar_field_renders_its_bare_type(self) -> None:+ assert _ddl_type(bigquery.SchemaField("clicks", "INTEGER")) == "INTEGER"++ def test_a_repeated_field_renders_as_an_array(self) -> None:+ # Dropping `mode` here would add the element type as a scalar column, which is+ # incompatible with the array BigQuery already autodetected for the scratch table.+ field = bigquery.SchemaField("tags", "STRING", mode="REPEATED")++ assert _ddl_type(field) == "ARRAY<STRING>"++ def test_a_record_field_renders_as_a_struct_of_its_subfields(self) -> None:+ # A bare `RECORD` is not valid DDL; the subfields must be spelled out.+ field = bigquery.SchemaField(+ "address",+ "RECORD",+ fields=[bigquery.SchemaField("city", "STRING"), bigquery.SchemaField("zip", "INTEGER")],+ )++ assert _ddl_type(field) == "STRUCT<`city` STRING, `zip` INTEGER>"++ def test_a_repeated_record_field_renders_as_an_array_of_structs(self) -> None:+ field = bigquery.SchemaField(+ "addresses", "RECORD", mode="REPEATED", fields=[bigquery.SchemaField("city", "STRING")]+ )++ assert _ddl_type(field) == "ARRAY<STRUCT<`city` STRING>>"++class TestCheckOwnedOrAbsent:def test_a_table_that_does_not_exist_is_reported_absent(self) -> None:writer = BigQueryDestinationWriter(_ctx())@@ -381,6 +411,29 @@ class TestIncremental:assert table.types["delivered_at"] == "INTEGER"assert sorted((row["id"], row["delivered_at"]) for row in table.rows) == [(1, 1700000500), (2, None)]+ async def test_a_column_a_later_batch_grows_is_added_to_the_target_before_merging(self) -> None:+ # MERGE does not evolve schema on its own, so a column only the second batch carries+ # must be added to the target first, or the MERGE fails referencing a column T lacks.+ client = FakeBigQueryClient()+ ctx = _ctx(sync_type="incremental", primary_keys=("id",))+ writer = LocalBigQueryWriter(ctx, client)++ await _deliver(writer, ctx, _rows([1], ["a"]), 0)+ await _deliver(+ writer,+ ctx,+ pa.RecordBatch.from_pydict({"id": [2], "name": ["b"], "clicks": [10]}),+ 1,+ is_final=True,+ )++ table = client.get_table("proj.dataset.charges")+ assert table.columns == ["id", "name", "clicks"]+ assert sorted((row["id"], row["name"], row.get("clicks")) for row in table.rows) == [+ (1, "a", None),+ (2, "b", 10),+ ]+async def test_text_that_cannot_convert_to_the_int64_column_is_a_configuration_error(self) -> None:# A retry sends the same values, so a retryable error would fail until someone changes the table.client = FakeBigQueryClient()diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.pyindex a8b54fb6af61..b08c2a68e8f0 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py@@ -109,6 +109,21 @@ def _bigquery_type_of(arrow_type: pa.DataType) -> str | None:return None+def _ddl_type(field: bigquery.SchemaField) -> str:+ """The DDL type declaration an `ALTER TABLE ADD COLUMN` needs for this field.++ `field.field_type` alone is only the element type: a `REPEATED` field needs `ARRAY<...>`+ around it, and a `RECORD`/`STRUCT` field needs its subfields spelled out, recursively for a+ nested struct or a struct inside an array.+ """+ base = (+ f"STRUCT<{', '.join(f'{_backtick(sub.name)} {_ddl_type(sub)}' for sub in field.fields)}>"+ if field.field_type in ("RECORD", "STRUCT")+ else field.field_type+ )+ return f"ARRAY<{base}>" if field.mode == "REPEATED" else base++class BigQueryDestinationConfigurationError(ValueError):"""The destination's config cannot produce a valid BigQuery write."""@@ -404,19 +419,20 @@ class BigQueryDestinationWriter:temp = f"{run.table_name}__ph_tmp_{run_scope(run.run_uuid, 8)}_{ctx.batch_index}_{chunk}"target_ref = self._table_ref(run.table_name)target = self._owned_table(client, target_ref, "merge into it")- target_absent = target is Noneif target is not None:table = self._align_to_target(table, target)self._load(client, temp, table, bigquery.WriteDisposition.WRITE_TRUNCATE)try:- if target_absent:+ if target is None:# `MERGE` fails outright against a target that does not exist, and an# incremental schema is on `sync_type` "incremental" from its very first run,# so nothing else ever creates it. Take the schema BigQuery just derived for the# scratch table rather than mapping Arrow types a second time.self._create_like(client, target_ref, temp)self._mark_owned(client, target_ref)+ else:+ self._evolve_table(client, target_ref, target.schema, temp, list(table.schema.names))self._merge(client, run.table_name, temp, list(table.schema.names), primary_keys)finally:client.delete_table(self._table_ref(temp), not_found_ok=True)@@ -451,6 +467,32 @@ class BigQueryDestinationWriter:table = table.set_column(index, pa.field(field.name, arrow_type), cast)return table+ def _evolve_table(+ self,+ client: bigquery.Client,+ target_ref: str,+ target_schema: list[bigquery.SchemaField],+ temp: str,+ column_names: list[str],+ ) -> None:+ """Add columns the source chunk has grown, before `MERGE` can reference them.++ Unlike `_load`'s `ALLOW_FIELD_ADDITION` on the plain-append path, `MERGE` does not+ evolve the target's schema on its own, so a column new to this chunk needs adding to+ the target first — otherwise the `MERGE` fails referencing a column T does not have.+ Additive only, same as the other SQL writers' `_evolve_table`. Skips the extra+ `get_table` call for the scratch table's schema when the chunk carries no column the+ target lacks, which is the common case on an incremental sync's steady-state batches.+ """+ existing = {field.name for field in target_schema}+ if all(name in existing for name in column_names):+ return+ additions = [field for field in client.get_table(self._table_ref(temp)).schema if field.name not in existing]+ if not additions:+ return+ clauses = ", ".join(f"ADD COLUMN IF NOT EXISTS {_backtick(f.name)} {_ddl_type(f)}" for f in additions)+ client.query(f"ALTER TABLE {_backtick(target_ref)} {clauses}").result()+def _create_like(self, client: bigquery.Client, table_ref: str, model: str) -> None:"""Create an empty table carrying another table's schema."""client.create_table(bigquery.Table(table_ref, schema=client.get_table(self._table_ref(model)).schema))