Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): align bigquery merge columns to the live table types (#118939)

posthog[bot]4ab05e627162
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
index 90b9fa81d4ae..d46aa42de604 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/fake_bigquery.py
@@ -5,8 +5,9 @@ re-applied or a finished run is replayed, so a test that only counted calls woul
This fake keeps rows and labels per table and applies the writer's load, copy, delete, merge and
drop-column statements to them, which lets a test assert the table's contents.
-It also refuses a `MERGE` whose source matches one target row twice, the way BigQuery does, so
-a source batch carrying a key more than once fails here too rather than passing quietly.
+It also refuses a `MERGE` whose source matches one target row twice, or whose source holds a
+column as a different type than the target, the way BigQuery does, so those batches fail here
+too rather than passing quietly.
"""
from __future__ import annotations
@@ -15,8 +16,9 @@ import re
from collections import Counter
from typing import Any
+import pyarrow as pa
import pyarrow.parquet as pq
-from google.api_core.exceptions import NotFound
+from google.api_core.exceptions import BadRequest, NotFound
from google.cloud import bigquery
Row = dict[str, Any]
@@ -32,24 +34,32 @@ _DROP_COLUMN = re.compile(
)
+# What schema autodetect gives a parquet column, in the legacy names `SchemaField` reports.
+_AUTODETECTED_TYPE = {pa.int64(): "INTEGER", pa.float64(): "FLOAT", pa.bool_(): "BOOLEAN"}
+
+
class MergeMultiMatchError(RuntimeError):
"""BigQuery's "UPDATE/MERGE must match at most one source row for each target row"."""
class FakeTable:
- def __init__(self, columns: list[str]) -> None:
- self.columns = list(columns)
+ def __init__(self, types: dict[str, str]) -> None:
+ self.types = dict(types)
self.rows: list[Row] = []
self.labels: dict[str, str] = {}
+ @property
+ def columns(self) -> list[str]:
+ return list(self.types)
+
@property
def schema(self) -> list[bigquery.SchemaField]:
- return [bigquery.SchemaField(name, "STRING") for name in self.columns]
+ return [bigquery.SchemaField(name, field_type) for name, field_type in self.types.items()]
- def add_columns(self, columns: list[str]) -> None:
- for column in columns:
- if column not in self.columns:
- self.columns.append(column)
+ def add_columns(self, types: dict[str, str]) -> None:
+ for column, field_type in types.items():
+ if column not in self.types:
+ self.types[column] = field_type
for row in self.rows:
row.setdefault(column, None)
@@ -81,7 +91,7 @@ class FakeBigQueryClient:
name = _ref_of(table)
if name in self.tables and not exists_ok:
raise ValueError(f"Already Exists: Table {name}")
- created = FakeTable([field.name for field in table.schema])
+ created = FakeTable({field.name: field.field_type for field in table.schema})
self.tables.setdefault(name, created)
return self.tables[name]
@@ -97,13 +107,13 @@ class FakeBigQueryClient:
def load_table_from_file(self, buffer, ref: str, job_config: bigquery.LoadJobConfig) -> _Job:
data = pq.read_table(buffer)
rows = data.to_pylist()
- columns = list(data.schema.names)
+ types = {field.name: _AUTODETECTED_TYPE.get(field.type, "STRING") for field in data.schema}
name = _ref_of(ref)
if job_config.write_disposition == bigquery.WriteDisposition.WRITE_TRUNCATE or name not in self.tables:
- self.tables[name] = FakeTable(columns)
+ self.tables[name] = FakeTable(types)
else:
- self.tables[name].add_columns(columns)
+ self.tables[name].add_columns(types)
table = self.tables[name]
for row in rows:
@@ -116,7 +126,7 @@ class FakeBigQueryClient:
# WRITE_TRUNCATE replaces the destination's data and schema but leaves its own labels
# alone, which is what lets the writer read its marker back after publishing.
labels = self.tables[name].labels if name in self.tables else {}
- copied = FakeTable(origin.columns)
+ copied = FakeTable(origin.types)
copied.rows = [dict(row) for row in origin.rows]
copied.labels = dict(labels)
self.tables[name] = copied
@@ -146,7 +156,7 @@ class FakeBigQueryClient:
table, column = match.group(1), match.group(2)
target = self.get_table(table)
if column in target.columns:
- target.columns.remove(column)
+ del target.types[column]
for row in target.rows:
row.pop(column, None)
return True
@@ -159,7 +169,12 @@ class FakeBigQueryClient:
target = self.get_table(tables.group(1))
source = self.get_table(tables.group(2))
keys = [target_key for target_key, _ in _MERGE_KEYS.findall(tables.group(3))]
- target.add_columns(source.columns)
+ for column, field_type in source.types.items():
+ if column in target.types and target.types[column] != field_type:
+ raise BadRequest(
+ f"Value of type {field_type} cannot be assigned to T.{column}, which has type {target.types[column]}"
+ )
+ target.add_columns(source.types)
incoming_per_key = Counter(_key_of(row, keys) for row in source.rows)
for row in target.rows:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
index d6ec01217301..46aacb47d139 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/tests/test_bigquery_writer.py
@@ -362,3 +362,32 @@ class TestIncremental:
"WHEN MATCHED THEN UPDATE SET T.`name` = S.`name` "
"WHEN NOT MATCHED THEN INSERT (`id`, `name`) VALUES (S.`id`, S.`name`)"
)
+
+ async def test_a_column_sent_as_text_merges_into_its_int64_column(self) -> None:
+ client = FakeBigQueryClient()
+ ctx = _ctx(sync_type="incremental", primary_keys=("id",))
+ writer = LocalBigQueryWriter(ctx, client)
+
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": [1700000000]}), 0)
+ await _deliver(
+ writer,
+ ctx,
+ pa.RecordBatch.from_pydict({"id": [1, 2], "delivered_at": ["1700000500", None]}),
+ 1,
+ is_final=True,
+ )
+
+ table = client.get_table("proj.dataset.charges")
+ assert table.types["delivered_at"] == "INTEGER"
+ assert sorted((row["id"], row["delivered_at"]) for row in table.rows) == [(1, 1700000500), (2, None)]
+
+ async def test_text_that_cannot_convert_to_the_int64_column_is_a_configuration_error(self) -> None:
+ # A retry sends the same values, so a retryable error would fail until someone changes the table.
+ client = FakeBigQueryClient()
+ ctx = _ctx(sync_type="incremental", primary_keys=("id",))
+ writer = LocalBigQueryWriter(ctx, client)
+
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": [1700000000]}), 0)
+
+ with pytest.raises(DestinationConfigurationError, match="delivered_at"):
+ await _deliver(writer, ctx, pa.RecordBatch.from_pydict({"id": [1], "delivered_at": ["soon"]}), 1)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
index 79a7869438fc..a8b54fb6af61 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/destinations_load/writers/bigquery.py
@@ -18,6 +18,7 @@ from contextlib import contextmanager
from typing import ClassVar
import pyarrow as pa
+import pyarrow.compute as pc
import pyarrow.parquet as pq
from asgiref.sync import async_to_sync, sync_to_async
from google.api_core.exceptions import Forbidden, NotFound
@@ -72,6 +73,42 @@ MISSING_PERMISSION_DETAIL = (
)
+# The Arrow type that loads into each BigQuery column type. A merge chunk whose column loads as a
+# different type is cast to it first, because a `MERGE` does not convert a STRING into an INT64.
+_ARROW_TYPE_FOR_BIGQUERY_TYPE: dict[str, pa.DataType] = {
+ "INT64": pa.int64(),
+ "FLOAT64": pa.float64(),
+ "BOOL": pa.bool_(),
+ "STRING": pa.string(),
+ "DATE": pa.date32(),
+ "TIME": pa.time64("us"),
+ "DATETIME": pa.timestamp("us"),
+ "TIMESTAMP": pa.timestamp("us", tz="UTC"),
+}
+
+# `SchemaField.field_type` can report the legacy SQL names.
+_STANDARD_BIGQUERY_TYPE = {"INTEGER": "INT64", "FLOAT": "FLOAT64", "BOOLEAN": "BOOL"}
+
+
+def _bigquery_type_of(arrow_type: pa.DataType) -> str | None:
+ """The BigQuery type that schema autodetect gives a parquet column of this Arrow type."""
+ if pa.types.is_integer(arrow_type):
+ return "INT64"
+ if pa.types.is_floating(arrow_type):
+ return "FLOAT64"
+ if pa.types.is_boolean(arrow_type):
+ return "BOOL"
+ if pa.types.is_string(arrow_type) or pa.types.is_large_string(arrow_type):
+ return "STRING"
+ if pa.types.is_date(arrow_type):
+ return "DATE"
+ if pa.types.is_time(arrow_type):
+ return "TIME"
+ if pa.types.is_timestamp(arrow_type):
+ return "TIMESTAMP" if arrow_type.tz else "DATETIME"
+ return None
+
+
class BigQueryDestinationConfigurationError(ValueError):
"""The destination's config cannot produce a valid BigQuery write."""
@@ -207,8 +244,8 @@ class BigQueryDestinationWriter:
# only holds characters a BigQuery label value accepts.
return hashlib.sha256(self._ctx.run_uuid.encode()).hexdigest()[:63]
- def _check_owned_or_absent(self, client: bigquery.Client, table_ref: str, action: str) -> bool:
- """Whether `table_ref` does not exist yet.
+ def _owned_table(self, client: bigquery.Client, table_ref: str, action: str) -> bigquery.Table | None:
+ """The table at `table_ref`, or None when it does not exist yet.
Raises if it exists and was not created by this schema's sync, rather than silently
truncating, merging into or appending to a table that predates this sync and merely
@@ -217,12 +254,16 @@ class BigQueryDestinationWriter:
try:
existing = client.get_table(table_ref)
except NotFound:
- return True
+ return None
if (existing.labels or {}).get(_OWNERSHIP_LABEL_KEY) != self._schema_label():
raise UnrelatedTableExistsError(
f"{table_ref} already exists and was not created by this sync; refusing to {action}."
)
- return False
+ return existing
+
+ def _check_owned_or_absent(self, client: bigquery.Client, table_ref: str, action: str) -> bool:
+ """Whether `table_ref` does not exist yet. Raises like `_owned_table`."""
+ return self._owned_table(client, table_ref, action) is None
def _set_labels(self, client: bigquery.Client, table_ref: str, labels: dict[str, str]) -> None:
existing = client.get_table(table_ref)
@@ -362,7 +403,10 @@ class BigQueryDestinationWriter:
temp = f"{run.table_name}__ph_tmp_{run_scope(run.run_uuid, 8)}_{ctx.batch_index}_{chunk}"
target_ref = self._table_ref(run.table_name)
- target_absent = self._check_owned_or_absent(client, target_ref, "merge into it")
+ target = self._owned_table(client, target_ref, "merge into it")
+ target_absent = target is None
+ if target is not None:
+ table = self._align_to_target(table, target)
self._load(client, temp, table, bigquery.WriteDisposition.WRITE_TRUNCATE)
try:
@@ -378,6 +422,35 @@ class BigQueryDestinationWriter:
client.delete_table(self._table_ref(temp), not_found_ok=True)
return table.num_rows
+ def _align_to_target(self, table: pa.Table, target: bigquery.Table) -> pa.Table:
+ """Cast each column to the type the live table holds it as.
+
+ The scratch table takes its schema from this chunk alone, but the live table keeps the
+ types of earlier runs. Upstream can send a column as text in one run, so without the cast
+ the `MERGE` fails on every retry. A value that cannot convert is a configuration error,
+ because only a change to the live table can let the chunk merge.
+ """
+ for field in target.schema:
+ if field.name not in table.schema.names or field.mode == "REPEATED":
+ continue
+ target_type = _STANDARD_BIGQUERY_TYPE.get(field.field_type, field.field_type)
+ arrow_type = _ARROW_TYPE_FOR_BIGQUERY_TYPE.get(target_type)
+ index = table.schema.get_field_index(field.name)
+ column = table.column(index)
+ if arrow_type is None or _bigquery_type_of(column.type) == target_type:
+ continue
+ try:
+ cast = pc.cast(column, arrow_type)
+ except pa.ArrowException as err:
+ raise DestinationConfigurationError(
+ self._ctx.destination_name,
+ f"The column {field.name} in {self._table_ref(self._ctx.table_name)} is {target_type}, but this "
+ f"sync sent values that cannot convert to {target_type}. Change the column type in BigQuery to "
+ "match the source, then run the sync again.",
+ ) from err
+ table = table.set_column(index, pa.field(field.name, arrow_type), cast)
+ return table
+
def _create_like(self, client: bigquery.Client, table_ref: str, model: str) -> None:
"""Create an empty table carrying another table's schema."""
client.create_table(bigquery.Table(table_ref, schema=client.get_table(self._table_ref(model)).schema))