:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): align bigquery merge columns to the live table types (#118939)
posthog[bot]4ab05e627162
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.pyindex 90b9fa81d4ae..d46aa42de604 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py@@ -5,8 +5,9 @@ re-applied or a finished run is replayed, so a test that only counted calls woulThis fake keeps rows and labels per table and applies the writer's load, copy, delete, merge anddrop-column statements to them, which lets a test assert the table's contents.-It also refuses a `MERGE` whose source matches one target row twice, the way BigQuery does, so-a source batch carrying a key more than once fails here too rather than passing quietly.+It also refuses a `MERGE` whose source matches one target row twice, or whose source holds a+column as a different type than the target, the way BigQuery does, so those batches fail here+too rather than passing quietly."""from __future__ import annotations@@ -15,8 +16,9 @@ import refrom collections import Counterfrom typing import Any+import pyarrow as paimport pyarrow.parquet as pq-from google.api_core.exceptions import NotFound+from google.api_core.exceptions import BadRequest, NotFoundfrom google.cloud import bigqueryRow = dict[str, Any]@@ -32,24 +34,32 @@ _DROP_COLUMN = re.compile()+# What schema autodetect gives a parquet column, in the legacy names `SchemaField` reports.+_AUTODETECTED_TYPE = {pa.int64(): "INTEGER", pa.float64(): "FLOAT", pa.bool_(): "BOOLEAN"}++class MergeMultiMatchError(RuntimeError):"""BigQuery's "UPDATE/MERGE must match at most one source row for each target row"."""class FakeTable:- def __init__(self, columns: list[str]) -> None:- self.columns = list(columns)+ def __init__(self, types: dict[str, str]) -> None:+ self.types = dict(types)self.rows: list[Row] = []self.labels: dict[str, str] = {}+ @property+ def columns(self) -> list[str]:+ return list(self.types)+@propertydef schema(self) -> list[bigquery.SchemaField]:- return [bigquery.SchemaField(name, "STRING") for name in self.columns]+ return [bigquery.SchemaField(name, field_type) for name, field_type in self.types.items()]- def add_columns(self, columns: list[str]) -> None:- for column in columns:- if column not in self.columns:- self.columns.append(column)+ def add_columns(self, types: dict[str, str]) -> None:+ for column, field_type in types.items():+ if column not in self.types:+ self.types[column] = field_typefor row in self.rows:row.setdefault(column, None)@@ -81,7 +91,7 @@ class FakeBigQueryClient:name = _ref_of(table)if name in self.tables and not exists_ok:raise ValueError(f"Already Exists: Table {name}")- created = FakeTable([field.name for field in table.schema])+ created = FakeTable({field.name: field.field_type for field in table.schema})self.tables.setdefault(name, created)return self.tables[name]@@ -97,13 +107,13 @@ class FakeBigQueryClient:def load_table_from_file(self, buffer, ref: str, job_config: bigquery.LoadJobConfig) -> _Job:data = pq.read_table(buffer)rows = data.to_pylist()- columns = list(data.schema.names)+ types = {field.name: _AUTODETECTED_TYPE.get(field.type, "STRING") for field in data.schema}name = _ref_of(ref)if job_config.write_disposition == bigquery.WriteDisposition.WRITE_TRUNCATE or name not in self.tables:- self.tables[name] = FakeTable(columns)+ self.tables[name] = FakeTable(types)else:- self.tables[name].add_columns(columns)+ self.tables[name].add_columns(types)table = self.tables[name]for row in rows:@@ -116,7 +126,7 @@ class FakeBigQueryClient:# WRITE_TRUNCATE replaces the destination's data and schema but leaves its own labels# alone, which is what lets the writer read its marker back after publishing.labels = self.tables[name].labels if name in self.tables else {}- copied = FakeTable(origin.columns)+ copied = FakeTable(origin.types)copied.rows = [dict(row) for row in origin.rows]copied.labels = dict(labels)self.tables[name] = copied@@ -146,7 +156,7 @@ class FakeBigQueryClient:table, column = match.group(1), match.group(2)target = self.get_table(table)if column in target.columns:- target.columns.remove(column)+ del target.types[column]for row in target.rows:row.pop(column, None)return True@@ -159,7 +169,12 @@ class FakeBigQueryClient:target = self.get_table(tables.group(1))source = self.get_table(tables.group(2))keys = [target_key for target_key, _ in _MERGE_KEYS.findall(tables.group(3))]- target.add_columns(source.columns)+ for column, field_type in source.types.items():+ if column in target.types and target.types[column] != field_type:+ raise BadRequest(+ f"Value of type {field_type} cannot be assigned to T.{column}, which has type {target.types[column]}"+ )+ target.add_columns(source.types)incoming_per_key = Counter(_key_of(row, keys) for row in source.rows)for row in target.rows:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.pyindex d6ec01217301..46aacb47d139 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py@@ -362,3 +362,32 @@ class TestIncremental:"WHEN MATCHED THEN UPDATE SET T.`name` = S.`name` ""WHEN NOT MATCHED THEN INSERT (`id`, `name`) VALUES (S.`id`, S.`name`)")++ async def test_a_column_sent_as_text_merges_into_its_int64_column(self) -> None:+ client = FakeBigQueryClient()+ ctx = _ctx(sync_type="incremental", primary_keys=("id",))+ writer = LocalBigQueryWriter(ctx, client)++ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": [1700000000]}), 0)+ await _deliver(+ writer,+ ctx,+ pa.RecordBatch.from_pydict({"id": [1, 2], "delivered_at": ["1700000500", None]}),+ 1,+ is_final=True,+ )++ table = client.get_table("proj.dataset.charges")+ assert table.types["delivered_at"] == "INTEGER"+ assert sorted((row["id"], row["delivered_at"]) for row in table.rows) == [(1, 1700000500), (2, None)]++ async def test_text_that_cannot_convert_to_the_int64_column_is_a_configuration_error(self) -> None:+ # A retry sends the same values, so a retryable error would fail until someone changes the table.+ client = FakeBigQueryClient()+ ctx = _ctx(sync_type="incremental", primary_keys=("id",))+ writer = LocalBigQueryWriter(ctx, client)++ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": [1700000000]}), 0)++ with pytest.raises(DestinationConfigurationError, match="delivered_at"):+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": ["soon"]}), 1)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.pyindex 79a7869438fc..a8b54fb6af61 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py@@ -18,6 +18,7 @@ from contextlib import contextmanagerfrom typing import ClassVarimport pyarrow as pa+import pyarrow.compute as pcimport pyarrow.parquet as pqfrom asgiref.sync import async_to_sync, sync_to_asyncfrom google.api_core.exceptions import Forbidden, NotFound@@ -72,6 +73,42 @@ MISSING_PERMISSION_DETAIL = ()+# The Arrow type that loads into each BigQuery column type. A merge chunk whose column loads as a+# different type is cast to it first, because a `MERGE` does not convert a STRING into an INT64.+_ARROW_TYPE_FOR_BIGQUERY_TYPE: dict[str, pa.DataType] = {+ "INT64": pa.int64(),+ "FLOAT64": pa.float64(),+ "BOOL": pa.bool_(),+ "STRING": pa.string(),+ "DATE": pa.date32(),+ "TIME": pa.time64("us"),+ "DATETIME": pa.timestamp("us"),+ "TIMESTAMP": pa.timestamp("us", tz="UTC"),+}++# `SchemaField.field_type` can report the legacy SQL names.+_STANDARD_BIGQUERY_TYPE = {"INTEGER": "INT64", "FLOAT": "FLOAT64", "BOOLEAN": "BOOL"}+++def _bigquery_type_of(arrow_type: pa.DataType) -> str | None:+ """The BigQuery type that schema autodetect gives a parquet column of this Arrow type."""+ if pa.types.is_integer(arrow_type):+ return "INT64"+ if pa.types.is_floating(arrow_type):+ return "FLOAT64"+ if pa.types.is_boolean(arrow_type):+ return "BOOL"+ if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type):+ return "STRING"+ if pa.types.is_date(arrow_type):+ return "DATE"+ if pa.types.is_time(arrow_type):+ return "TIME"+ if pa.types.is_timestamp(arrow_type):+ return "TIMESTAMP" if arrow_type.tz else "DATETIME"+ return None++class BigQueryDestinationConfigurationError(ValueError):"""The destination's config cannot produce a valid BigQuery write."""@@ -207,8 +244,8 @@ class BigQueryDestinationWriter:# only holds characters a BigQuery label value accepts.return hashlib.sha256(self._ctx.run_uuid.encode()).hexdigest()[:63]- def _check_owned_or_absent(self, client: bigquery.Client, table_ref: str, action: str) -> bool:- """Whether `table_ref` does not exist yet.+ def _owned_table(self, client: bigquery.Client, table_ref: str, action: str) -> bigquery.Table | None:+ """The table at `table_ref`, or None when it does not exist yet.Raises if it exists and was not created by this schema's sync, rather than silentlytruncating, merging into or appending to a table that predates this sync and merely@@ -217,12 +254,16 @@ class BigQueryDestinationWriter:try:existing = client.get_table(table_ref)except NotFound:- return True+ return Noneif (existing.labels or {}).get(_OWNERSHIP_LABEL_KEY) != self._schema_label():raise UnrelatedTableExistsError(f"{table_ref} already exists and was not created by this sync; refusing to {action}.")- return False+ return existing++ def _check_owned_or_absent(self, client: bigquery.Client, table_ref: str, action: str) -> bool:+ """Whether `table_ref` does not exist yet. Raises like `_owned_table`."""+ return self._owned_table(client, table_ref, action) is Nonedef _set_labels(self, client: bigquery.Client, table_ref: str, labels: dict[str, str]) -> None:existing = client.get_table(table_ref)@@ -362,7 +403,10 @@ class BigQueryDestinationWriter:temp = f"{run.table_name}__ph_tmp_{run_scope(run.run_uuid, 8)}_{ctx.batch_index}_{chunk}"target_ref = self._table_ref(run.table_name)- target_absent = self._check_owned_or_absent(client, target_ref, "merge into it")+ target = self._owned_table(client, target_ref, "merge into it")+ target_absent = target is None+ if target is not None:+ table = self._align_to_target(table, target)self._load(client, temp, table, bigquery.WriteDisposition.WRITE_TRUNCATE)try:@@ -378,6 +422,35 @@ class BigQueryDestinationWriter:client.delete_table(self._table_ref(temp), not_found_ok=True)return table.num_rows+ def _align_to_target(self, table: pa.Table, target: bigquery.Table) -> pa.Table:+ """Cast each column to the type the live table holds it as.++ The scratch table takes its schema from this chunk alone, but the live table keeps the+ types of earlier runs. Upstream can send a column as text in one run, so without the cast+ the `MERGE` fails on every retry. A value that cannot convert is a configuration error,+ because only a change to the live table can let the chunk merge.+ """+ for field in target.schema:+ if field.name not in table.schema.names or field.mode == "REPEATED":+ continue+ target_type = _STANDARD_BIGQUERY_TYPE.get(field.field_type, field.field_type)+ arrow_type = _ARROW_TYPE_FOR_BIGQUERY_TYPE.get(target_type)+ index = table.schema.get_field_index(field.name)+ column = table.column(index)+ if arrow_type is None or _bigquery_type_of(column.type) == target_type:+ continue+ try:+ cast = pc.cast(column, arrow_type)+ except pa.ArrowException as err:+ raise DestinationConfigurationError(+ self._ctx.destination_name,+ f"The column {field.name} in {self._table_ref(self._ctx.table_name)} is {target_type}, but this "+ f"sync sent values that cannot convert to {target_type}. Change the column type in BigQuery to "+ "match the source, then run the sync again.",+ ) from err+ table = table.set_column(index, pa.field(field.name, arrow_type), cast)+ return table+def _create_like(self, client: bigquery.Client, table_ref: str, model: str) -> None:"""Create an empty table carrying another table's schema."""client.create_table(bigquery.Table(table_ref, schema=client.get_table(self._table_ref(model)).schema))