:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-imports): align bigquery append batches to the target's column types (#126052)
Tom Owers325bc19e0d37
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.pyindex 692d8f22184b..dfc600cfc013 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py@@ -6,8 +6,9 @@ This fake keeps rows and labels per table and applies the writer's load, copy, ddrop-column statements to them, which lets a test assert the table's contents.It also refuses a `MERGE` whose source matches one target row twice, whose source holds a column-the target does not have, or whose source holds a column as a different type than the target, the-way BigQuery does, so those batches fail here too rather than passing quietly.+the target does not have, or whose source holds a column as a different type than the target, and+refuses an append load job that sends an existing column as a different type, the way BigQuery+does, so those batches fail here too rather than passing quietly."""from __future__ import annotations@@ -123,7 +124,14 @@ class FakeBigQueryClient:if job_config.write_disposition == bigquery.WriteDisposition.WRITE_TRUNCATE or name not in self.tables:self.tables[name] = FakeTable(types)else:- self.tables[name].add_columns(types)+ existing = self.tables[name]+ for column, field_type in types.items():+ if column in existing.types and existing.types[column] != field_type:+ raise BadRequest(+ f"Provided Schema does not match Table {name}. Field {column} has changed type "+ f"from {existing.types[column]} to {field_type}"+ )+ existing.add_columns(types)table = self.tables[name]for row in rows:diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.pyindex 34765baaabf2..45fd00b0e0fe 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py@@ -479,3 +479,50 @@ class TestIncremental:with pytest.raises(DestinationConfigurationError, match="delivered_at"):await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": ["soon"]}), 1)+++class TestAppend:+ """sync_type "incremental" with no primary keys: plain `WRITE_APPEND`, no `MERGE`."""++ async def test_a_whole_number_float_appends_without_changing_the_int64_column(self) -> None:+ # Regression: the append path used to load the raw batch straight through, so a column+ # BigQuery had autodetected as INTEGER on an earlier batch failed the whole load job as+ # soon as a later batch sent it as a float, even when every value still fit as a whole+ # number. The merge path already aligned to the target's type before this writer existed+ # for plain appends.+ client = FakeBigQueryClient()+ ctx = _ctx(sync_type="incremental")+ writer = LocalBigQueryWriter(ctx, client)++ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "count": [1]}), 0)+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [2], "count": [2.0]}), 1, is_final=True)++ table = client.get_table("proj.dataset.charges")+ assert table.types["count"] == "INTEGER"+ assert sorted(row["count"] for row in table.rows) == [1, 2]++ async def test_a_fractional_value_that_cannot_fit_the_int64_column_is_a_configuration_error(self) -> None:+ # A retry sends the same values, so a retryable error would fail until someone widens+ # the column in BigQuery to FLOAT64.+ client = FakeBigQueryClient()+ ctx = _ctx(sync_type="incremental")+ writer = LocalBigQueryWriter(ctx, client)++ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "rate": [1]}), 0)++ with pytest.raises(DestinationConfigurationError, match="rate"):+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [2], "rate": [0.5]}), 1)++ async def test_a_number_other_than_0_or_1_cannot_convert_to_the_bool_column(self) -> None:+ # Regression: `pc.cast` treats any nonzero number as truthy rather than raising, unlike+ # its float-to-int64 cast, so aligning to a BOOL target used to silently turn 2 into true+ # instead of raising the way a direct append to BigQuery always did.+ client = FakeBigQueryClient()+ ctx = _ctx(sync_type="incremental")+ writer = LocalBigQueryWriter(ctx, client)++ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "active": [True]}), 0)++ with pytest.raises(DestinationConfigurationError, match="active"):+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [2], "active": [2]}), 1)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.pyindex b2a62b709f7f..7fc51d0e25c8 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py@@ -368,9 +368,15 @@ class BigQueryDestinationWriter:# pq.write_table needs a Table, and one record batch is one load job.table = pa.Table.from_batches([record_batch])target_ref = self._table_ref(run.table_name)- target_absent = self._check_owned_or_absent(client, target_ref, "append to it")+ target = self._owned_table(client, target_ref, "append to it")+ if target is not None:+ # `ALLOW_FIELD_ADDITION` only covers a column the target lacks; it does not let+ # a load job widen a column's existing type, so a batch sending a wider type+ # than the target already has (e.g. a count turning into a fraction) fails the+ # load outright on every retry without this.+ table = self._align_to_target(table, target)self._load(client, run.table_name, table, bigquery.WriteDisposition.WRITE_APPEND)- if target_absent:+ if target is None:self._mark_owned(client, target_ref)return table.num_rows@@ -461,6 +467,12 @@ class BigQueryDestinationWriter:continuetry:cast = pc.cast(column, arrow_type)+ # `safe` casting still lets any nonzero number through to BOOL as `true`, because+ # Arrow does not treat that as a loss of precision the way it does a fractional+ # float truncated to an int. Casting back and comparing catches it the same way a+ # retry against the live table would have, instead of silently losing the number.+ if arrow_type == pa.bool_() and not column.equals(pc.cast(cast, column.type)):+ raise pa.ArrowInvalid(f"{field.name} holds a value other than 0 or 1")except pa.ArrowException as err:raise DestinationConfigurationError(self._ctx.destination_name,