Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-imports): align bigquery append batches to the target's column types (#126052)

Tom Owers325bc19e0d37
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
index 692d8f22184b..dfc600cfc013 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
@@ -6,8 +6,9 @@ This fake keeps rows and labels per table and applies the writer's load, copy, d
drop-column statements to them, which lets a test assert the table's contents.
It also refuses a `MERGE` whose source matches one target row twice, whose source holds a column
-the target does not have, or whose source holds a column as a different type than the target, the
-way BigQuery does, so those batches fail here too rather than passing quietly.
+the target does not have, or whose source holds a column as a different type than the target, and
+refuses an append load job that sends an existing column as a different type, the way BigQuery
+does, so those batches fail here too rather than passing quietly.
"""
from __future__ import annotations
@@ -123,7 +124,14 @@ class FakeBigQueryClient:
if job_config.write_disposition == bigquery.WriteDisposition.WRITE_TRUNCATE or name not in self.tables:
self.tables[name] = FakeTable(types)
else:
- self.tables[name].add_columns(types)
+ existing = self.tables[name]
+ for column, field_type in types.items():
+ if column in existing.types and existing.types[column] != field_type:
+ raise BadRequest(
+ f"Provided Schema does not match Table {name}. Field {column} has changed type "
+ f"from {existing.types[column]} to {field_type}"
+ )
+ existing.add_columns(types)
table = self.tables[name]
for row in rows:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
index 34765baaabf2..45fd00b0e0fe 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
@@ -479,3 +479,50 @@ class TestIncremental:
with pytest.raises(DestinationConfigurationError, match="delivered_at"):
await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": ["soon"]}), 1)
+
+
+class TestAppend:
+ """sync_type "incremental" with no primary keys: plain `WRITE_APPEND`, no `MERGE`."""
+
+ async def test_a_whole_number_float_appends_without_changing_the_int64_column(self) -> None:
+ # Regression: the append path used to load the raw batch straight through, so a column
+ # BigQuery had autodetected as INTEGER on an earlier batch failed the whole load job as
+ # soon as a later batch sent it as a float, even when every value still fit as a whole
+ # number. The merge path already aligned to the target's type before this writer existed
+ # for plain appends.
+ client = FakeBigQueryClient()
+ ctx = _ctx(sync_type="incremental")
+ writer = LocalBigQueryWriter(ctx, client)
+
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "count": [1]}), 0)
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [2], "count": [2.0]}), 1, is_final=True)
+
+ table = client.get_table("proj.dataset.charges")
+ assert table.types["count"] == "INTEGER"
+ assert sorted(row["count"] for row in table.rows) == [1, 2]
+
+ async def test_a_fractional_value_that_cannot_fit_the_int64_column_is_a_configuration_error(self) -> None:
+ # A retry sends the same values, so a retryable error would fail until someone widens
+ # the column in BigQuery to FLOAT64.
+ client = FakeBigQueryClient()
+ ctx = _ctx(sync_type="incremental")
+ writer = LocalBigQueryWriter(ctx, client)
+
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "rate": [1]}), 0)
+
+ with pytest.raises(DestinationConfigurationError, match="rate"):
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [2], "rate": [0.5]}), 1)
+
+ async def test_a_number_other_than_0_or_1_cannot_convert_to_the_bool_column(self) -> None:
+ # Regression: `pc.cast` treats any nonzero number as truthy rather than raising, unlike
+ # its float-to-int64 cast, so aligning to a BOOL target used to silently turn 2 into true
+ # instead of raising the way a direct append to BigQuery always did.
+ client = FakeBigQueryClient()
+ ctx = _ctx(sync_type="incremental")
+ writer = LocalBigQueryWriter(ctx, client)
+
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "active": [True]}), 0)
+
+ with pytest.raises(DestinationConfigurationError, match="active"):
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [2], "active": [2]}), 1)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
index b2a62b709f7f..7fc51d0e25c8 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
@@ -368,9 +368,15 @@ class BigQueryDestinationWriter:
# pq.write_table needs a Table, and one record batch is one load job.
table = pa.Table.from_batches([record_batch])
target_ref = self._table_ref(run.table_name)
- target_absent = self._check_owned_or_absent(client, target_ref, "append to it")
+ target = self._owned_table(client, target_ref, "append to it")
+ if target is not None:
+ # `ALLOW_FIELD_ADDITION` only covers a column the target lacks; it does not let
+ # a load job widen a column's existing type, so a batch sending a wider type
+ # than the target already has (e.g. a count turning into a fraction) fails the
+ # load outright on every retry without this.
+ table = self._align_to_target(table, target)
self._load(client, run.table_name, table, bigquery.WriteDisposition.WRITE_APPEND)
- if target_absent:
+ if target is None:
self._mark_owned(client, target_ref)
return table.num_rows
@@ -461,6 +467,12 @@ class BigQueryDestinationWriter:
continue
try:
cast = pc.cast(column, arrow_type)
+ # `safe` casting still lets any nonzero number through to BOOL as `true`, because
+ # Arrow does not treat that as a loss of precision the way it does a fractional
+ # float truncated to an int. Casting back and comparing catches it the same way a
+ # retry against the live table would have, instead of silently losing the number.
+ if arrow_type == pa.bool_() and not column.equals(pc.cast(cast, column.type)):
+ raise pa.ArrowInvalid(f"{field.name} holds a value other than 0 or 1")
except pa.ArrowException as err:
raise DestinationConfigurationError(
self._ctx.destination_name,