Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): refuse every column name a cdc sync writes (#123543)

Daniel RC7ac6bc808335
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.py
index 3e36a91f6b1d..2ca5d6b15764 100644
--- a/products/warehouse_sources/backend/facade/source_management.py
+++ b/products/warehouse_sources/backend/facade/source_management.py
@@ -29,9 +29,9 @@ _LAZY = {
"SELF_MANAGED_LAG_REASON": "cdc.broken",
"tables_wait_for_repair": "cdc.broken",
"purge_buffer_prefix": "cdc.buffer",
- "CDC_SEQ_COLUMN": "cdc.batcher",
"is_history_table_mode": "cdc.batcher",
- "history_column_collisions": "cdc.batcher",
+ "reserved_column_collisions": "cdc.batcher",
+ "reserved_column_names": "cdc.batcher",
"BUFFER_LANE": "cdc.snapshot_lane",
"CDC_RESET_PENDING_KEY": "cdc.snapshot_lane",
"hand_reset_to_capture_if_sync_running": "cdc.snapshot_lane",
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 2a3a82d170f4..067527870a35 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -66,7 +66,6 @@ from products.warehouse_sources.backend.facade.source_inputs import SourceInputs
from products.warehouse_sources.backend.facade.source_management import (
BUFFER_LANE,
CDC_RESET_PENDING_KEY,
- CDC_SEQ_COLUMN,
AnySource,
RowFilterValidationError,
SourceRegistry,
@@ -75,10 +74,11 @@ from products.warehouse_sources.backend.facade.source_management import (
filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,
get_cdc_adapter,
hand_reset_to_capture_if_sync_running,
- history_column_collisions,
is_history_table_mode,
purge_buffer_prefix,
repair_is_running,
+ reserved_column_collisions,
+ reserved_column_names,
resnapshot_stays_in_buffer,
source_type_supports_cdc,
tables_wait_for_repair,
@@ -457,17 +457,7 @@ def _source_column_names(instance: ExternalDataSchema) -> set[str]:
return {column["name"] for column in columns if isinstance(column, dict) and isinstance(column.get("name"), str)}
-def _refuse_reserved_cdc_column(instance: ExternalDataSchema) -> None:
- """Capture stamps each change with this column, so a source column of the same name would fail
- the source's sync."""
- if CDC_SEQ_COLUMN in _source_column_names(instance):
- raise ValidationError(
- "Change data capture can't sync a column named _ph_cdc_seq, because PostHog uses that name. "
- "Rename the column on your database, or choose another sync method for this table."
- )
-
-
-def _refuse_reserved_history_columns(
+def _refuse_reserved_columns(
instance: ExternalDataSchema,
*,
previous_mode: str | None,
@@ -477,23 +467,31 @@ def _refuse_reserved_history_columns(
new_enabled_columns: list[str] | None,
new_primary_keys: list[str] | None,
) -> None:
- """The history table stamps each row version with `valid_from` and `valid_to`, so a synced source
- column of the same name fails every load into it. Only a collision this request adds is refused,
- so a table that already has one can still be edited or switched back."""
- if not is_history_table_mode(new_mode):
- return
+ """Capture and the history table write their own columns next to the source's, so a synced source
+ column that loads under one of those names fails the sync. Only a collision this request adds is
+ refused, so a table that already has one can still be edited or switched back."""
columns = _source_column_names(instance)
- taken = history_column_collisions(columns, new_enabled_columns, new_primary_keys, instance.incremental_field)
- if instance.sync_type == ExternalDataSchema.SyncType.CDC and is_history_table_mode(previous_mode):
- existing = history_column_collisions(
- columns, previous_enabled_columns, previous_primary_keys, instance.incremental_field
+ taken = reserved_column_collisions(
+ columns,
+ reserved_column_names(history=is_history_table_mode(new_mode)),
+ enabled_columns=new_enabled_columns,
+ primary_keys=new_primary_keys,
+ incremental_field=instance.incremental_field,
+ )
+ if taken and instance.sync_type == ExternalDataSchema.SyncType.CDC:
+ existing = reserved_column_collisions(
+ columns,
+ reserved_column_names(history=is_history_table_mode(previous_mode)),
+ enabled_columns=previous_enabled_columns,
+ primary_keys=previous_primary_keys,
+ incremental_field=instance.incremental_field,
)
taken = [column for column in taken if column not in existing]
if taken:
raise ValidationError(
- f"A history table can't store the column {' or '.join(taken)}, because PostHog uses valid_from and "
- "valid_to to record when each version was current. Rename the column on your database, or choose "
- "the consolidated table only for this table."
+ f"Change data capture can't sync the column {' or '.join(taken)}, because it loads under a name "
+ "PostHog writes. Rename the column on your database, leave it out of the synced columns, or change "
+ "this table's sync method or output tables."
)
@@ -1331,7 +1329,6 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
# discovery already stored; refuse the switch when neither is set.
_apply_primary_key_columns(data, payload, instance, "CDC")
if instance.sync_type != ExternalDataSchema.SyncType.CDC:
- _refuse_reserved_cdc_column(instance)
# A table joining CDC starts from a fresh snapshot, even when an earlier sync
# method or an earlier CDC period already loaded it. The snapshot moves to
# streaming only when initial_sync_complete goes from False to True, so a table
@@ -1369,7 +1366,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
# can also bring a validity column into the history table.
if (sync_type if "sync_type" in data else instance.sync_type) == ExternalDataSchema.SyncType.CDC:
resulting_config = validated_data.get("sync_type_config", instance.sync_type_config) or {}
- _refuse_reserved_history_columns(
+ _refuse_reserved_columns(
instance,
previous_mode=previous_cdc_table_mode,
previous_enabled_columns=instance.enabled_columns,
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index f360c4657e69..7869fab3340b 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -51,7 +51,6 @@ from products.warehouse_sources.backend.facade.models import (
)
from products.warehouse_sources.backend.facade.source_inputs import SourceInputsStore, UnreadableSourceInputsError
from products.warehouse_sources.backend.facade.source_management import (
- CDC_SEQ_COLUMN,
PREVIEW_DEFAULT_ROWS,
PREVIEW_MAX_ROWS,
AnySource,
@@ -72,10 +71,11 @@ from products.warehouse_sources.backend.facade.source_management import (
fetch_docs_text,
filter_dwh_columns_by_enabled_columns,
get_cdc_adapter,
- history_column_collisions,
is_history_table_mode,
is_team_allowlisted_for_internal_hosts,
new_source_requires_ssl,
+ reserved_column_collisions,
+ reserved_column_names,
sql_schema_metadata,
validate_and_coerce_row_filters,
)
@@ -1454,18 +1454,23 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
},
)
- # Capture stamps each change with this column, so a source column of the same name would
- # fail the source's first sync. Refuse before any replication state exists.
+ # Capture and the history table write their own columns next to the source's, so a synced
+ # source column that loads under one of those names would fail the source's first sync.
+ # Refuse before any replication state exists.
+ reserved_columns_by_table = {
+ schema["name"]: reserved_column_collisions(
+ [column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],
+ reserved_column_names(history=is_history_table_mode(schema.get("cdc_table_mode"))),
+ enabled_columns=_payload_enabled_columns(schema),
+ primary_keys=pk_columns_by_table.get(schema["name"]),
+ )
+ for schema in payload_schemas
+ if schema.get("sync_type") == "cdc"
+ and schema.get("should_sync", False)
+ and isinstance(schema.get("name"), str)
+ }
tables_with_reserved_column = sorted(
- {
- schema["name"]
- for schema in payload_schemas
- if schema.get("sync_type") == "cdc"
- and schema.get("should_sync", False)
- and isinstance(schema.get("name"), str)
- and CDC_SEQ_COLUMN
- in {column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])}
- }
+ f"{table} ({', '.join(columns)})" for table, columns in reserved_columns_by_table.items() if columns
)
if tables_with_reserved_column:
new_source_model.delete()
@@ -1473,40 +1478,9 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
status=status.HTTP_400_BAD_REQUEST,
data={
"message": (
- "Change data capture can't sync a column named _ph_cdc_seq, because PostHog uses "
- "that name. Rename the column on your database, or choose another sync method for "
- f"these tables: {', '.join(tables_with_reserved_column)}."
- )
- },
- )
-
- # The history table stamps each row version with these columns, so a synced source column
- # of the same name would fail every load into it.
- tables_with_validity_column = sorted(
- {
- schema["name"]
- for schema in payload_schemas
- if schema.get("sync_type") == "cdc"
- and schema.get("should_sync", False)
- and is_history_table_mode(schema.get("cdc_table_mode"))
- and isinstance(schema.get("name"), str)
- and history_column_collisions(
- [column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],
- _payload_enabled_columns(schema),
- pk_columns_by_table.get(schema["name"]),
- )
- }
- )
- if tables_with_validity_column:
- new_source_model.delete()
- return Response(
- status=status.HTTP_400_BAD_REQUEST,
- data={
- "message": (
- "A history table can't store a column that loads as valid_from or valid_to, because "
- "PostHog uses those names to record when each version was current. Rename the column on "
- "your database, or choose the consolidated table only for these tables: "
- f"{', '.join(tables_with_validity_column)}."
+ "Change data capture can't sync a column that loads under a name PostHog writes. Rename "
+ "the column on your database, leave it out of the synced columns, or change the sync "
+ f"method or output tables for these tables: {'; '.join(tables_with_reserved_column)}."
)
},
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
index 7badfcdd00ee..d79c5e7e0d08 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
@@ -46,9 +46,12 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters impor
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_SEQ_COLUMN,
+ HISTORY_COLUMNS,
ChangeEventBatcher,
enrich_delete_rows,
enrich_toast_omitted_rows,
+ is_history_table_mode,
+ reserved_column_collisions,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.billing_expiry import (
blocked_past_buffer_retention,
@@ -350,17 +353,25 @@ class CDCExtractActivity:
# ------------------------------------------------------------------
# Per-flush processing
# ------------------------------------------------------------------
+ def _writes_buffer_for(self, table_name: str) -> bool:
+ return (
+ table_name in self.schema_by_name
+ and table_name in self._buffered_table_names
+ and table_name not in self._tables_awaiting_reset
+ )
+
def _process_flush(self, tables: dict[str, pa.Table]) -> None:
"""Enrich one micro-batch and write each captured table's changes to the S3 change buffer."""
events_extracted = 0
+ assert self.batcher is not None
+ for table_name, error in self.batcher.reserved_collisions.items():
+ if self._writes_buffer_for(table_name):
+ raise error
+
for table_name, raw_table in tables.items():
schema = self.schema_by_name.get(table_name)
- if (
- schema is None
- or table_name not in self._buffered_table_names
- or table_name in self._tables_awaiting_reset
- ):
+ if schema is None or not self._writes_buffer_for(table_name):
continue
self._safe_heartbeat()
@@ -368,6 +379,13 @@ class CDCExtractActivity:
# raw_table has one row per source change event.
events_extracted += raw_table.num_rows
+ # The batcher already refused the capture columns. The history table's own columns are
+ # checked here, because only the schema says whether this table keeps history.
+ if is_history_table_mode(schema.cdc_table_mode):
+ taken = reserved_column_collisions(raw_table.column_names, HISTORY_COLUMNS)
+ if taken:
+ raise CDCReservedColumnError(f"Table {table_name} has history table column(s) {taken}")
+
key_columns = self.pk_columns_by_table.get(table_name, [])
# TOAST fill first so DELETE enrichment copies resolved values, not the
@@ -377,11 +395,10 @@ class CDCExtractActivity:
if not enriched_table.num_rows:
continue
- # A source column named _ph_cdc_seq means the batcher could not append the
- # engine position; writing anyway would name, order, and clean up files by
- # customer data — cleanup can then delete unconsumed files (see errors.py).
+ # The batcher refuses a source column of this name, so only a capture bug gets here. Writing
+ # without the engine position could make cleanup delete unconsumed files (see errors.py).
if not has_engine_seq(enriched_table):
- raise CDCReservedColumnError(f"Table {table_name} has a source column named {CDC_SEQ_COLUMN}")
+ raise RuntimeError(f"Table {table_name} reached the buffer without the engine position")
self._write_buffer_file(schema, table_name, enriched_table)
if events_extracted:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
index 278a583313ad..2646fe7f0d96 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
@@ -21,7 +21,7 @@ DELETED_AT_COLUMN = "_ph_deleted_at"
# and (in Phase B) drives the merge's monotonicity guard.
CDC_SEQ_COLUMN = "_ph_cdc_seq"
# Marks CDC_SEQ_COLUMN as engine-produced, so readers can tell it from a source column of the same
-# name (see the collision skip in _events_to_table).
+# name.
CDC_SEQ_PROVENANCE = {b"posthog_cdc": b"engine_position"}
# Suffix of the SCD2 companion table's resource name ({schema.name}_cdc).
@@ -57,22 +57,36 @@ def is_history_table_mode(cdc_table_mode: object) -> bool:
return isinstance(cdc_table_mode, str) and cdc_table_mode in CDC_HISTORY_TABLE_MODES
-def history_column_collisions(
+# Every column capture stamps onto a change row. The consolidated table and the history table both store them.
+CAPTURE_COLUMNS: frozenset[str] = frozenset(
+ {CDC_OP_COLUMN, CDC_TIMESTAMP_COLUMN, CDC_SEQ_COLUMN, DELETED_COLUMN, DELETED_AT_COLUMN, TOAST_OMITTED_COLUMN}
+)
+# The columns that only the history table adds.
+HISTORY_COLUMNS: frozenset[str] = frozenset({SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN})
+
+
+def reserved_column_names(*, history: bool) -> frozenset[str]:
+ """Every name a CDC sync writes next to the source's own columns."""
+ return CAPTURE_COLUMNS | HISTORY_COLUMNS if history else CAPTURE_COLUMNS
+
+
+def reserved_column_collisions(
column_names: Iterable[str],
- enabled_columns: list[str] | None,
- primary_keys: list[str] | None,
+ reserved: frozenset[str],
+ *,
+ enabled_columns: list[str] | None = None,
+ primary_keys: list[str] | None = None,
incremental_field: str | None = None,
) -> list[str]:
- """Source columns that the history table would receive under its validity column names."""
+ """Source columns that would load under a name in `reserved`."""
synced = set(column_names)
# The same projection capture applies (`filter_columns_by_enabled_columns`), kept inline because
# that module's package imports Django models and this one sits on the capture path.
if enabled_columns is not None:
synced &= {*enabled_columns, *(primary_keys or []), *([incremental_field] if incremental_field else [])}
- # The load renames columns to snake_case before the history transform adds the validity columns,
- # so `validFrom` collides too. The source names are returned, because those are what a user renames.
- validity_columns = {SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN}
- return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in validity_columns)
+ # The load renames columns to snake_case, so `validFrom` and `_PH_CDC_SEQ` collide too. The source
+ # names are returned, because those are what a user renames.
+ return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in reserved)
# Columns that are CDC/SCD2 metadata — always filled from the event itself,
@@ -124,6 +138,10 @@ class ChangeEventBatcher:
# Engine adapter's position_serialized → int converter. When set, flushed
# tables carry CDC_SEQ_COLUMN; when None the column is absent entirely.
self._position_to_seq = position_to_seq
+ # Tables the last flush could not build, because a source column collides with a capture column.
+ # The caller raises for them only once it knows the table is not waiting for a reset, because a
+ # reset discards the table's changes.
+ self.reserved_collisions: dict[str, CDCReservedColumnError] = {}
def add(self, event: ChangeEvent) -> None:
size = self._estimate_event_bytes(event)
@@ -142,11 +160,15 @@ class ChangeEventBatcher:
Returns empty dict if no events buffered.
"""
result: dict[str, pa.Table] = {}
+ self.reserved_collisions = {}
for table_name, events in self._events.items():
if not events:
continue
- result[table_name] = _events_to_table(events, position_to_seq=self._position_to_seq)
+ try:
+ result[table_name] = _events_to_table(events, position_to_seq=self._position_to_seq)
+ except CDCReservedColumnError as error:
+ self.reserved_collisions[table_name] = error
self._events.clear()
self._table_bytes.clear()
@@ -453,13 +475,11 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:
A two-step merge + append in the load processor closes previous "current"
rows (sets valid_to) when a new batch is written for the same PK.
"""
- taken = {SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN} & set(pa_table.column_names)
+ taken = reserved_column_collisions(pa_table.column_names, HISTORY_COLUMNS)
if taken:
# Delta refuses a duplicate column name at write time; failing here names the column and
# keeps a half-built batch out of the writer.
- raise CDCReservedColumnError(
- f"Source column(s) {sorted(taken)} collide with the history table's validity columns"
- )
+ raise CDCReservedColumnError(f"Source column(s) {taken} collide with the history table's validity columns")
# Taken from the batch rather than assumed: `valid_from` is the timestamp column's own values,
# and declaring a type it does not have makes pyarrow reject the append outright.
ts_type = (
@@ -517,6 +537,12 @@ def _events_to_table(events: list[ChangeEvent], position_to_seq: Callable[[str],
for col_name in event.omitted_columns:
all_columns[col_name] = None
column_names = list(all_columns.keys())
+ taken = reserved_column_collisions(column_names, CAPTURE_COLUMNS)
+ if taken:
+ raise CDCReservedColumnError(
+ f"Table {events[0].table_name} has source column(s) {taken} that collide with the columns change data "
+ "capture writes"
+ )
# Authoritative Arrow type per column from the source schema (same for every event
# of a relation). Lets all-null micro-batches keep the column's real type.
@@ -583,9 +609,7 @@ def _events_to_table(events: list[ChangeEvent], position_to_seq: Callable[[str],
arrays.append(pa.array(omitted_lists, type=pa.list_(pa.string())))
fields.append(pa.field(TOAST_OMITTED_COLUMN, pa.list_(pa.string())))
- # Skip on collision: a source column literally named _ph_cdc_seq is never overwritten. The batch
- # then carries no engine seq, and capture refuses to write it to the buffer.
- if position_to_seq is not None and CDC_SEQ_COLUMN not in column_names:
+ if position_to_seq is not None:
seq_values = [position_to_seq(event.position_serialized) for event in events]
arrays.append(pa.array(seq_values, type=pa.int64()))
fields.append(pa.field(CDC_SEQ_COLUMN, pa.int64(), metadata=CDC_SEQ_PROVENANCE))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
index 291222b0d500..793fce39f5d3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
@@ -74,14 +74,13 @@ class CDCReservedColumnError(Exception):
Non-retryable: the collision is a property of the customer's table, so replaying re-fails.
- For the engine position column it is raised when capture writes the buffer — buffer files
- derive their name, ordering, and retry cleanup from it, and a same-named source column means
- the batcher could not append it. Writing anyway would order and clean up by customer data,
- which can silently delete unconsumed buffer files.
-
- For the history table's validity columns it is raised on every path, from the SCD2 stamp
- itself: Delta refuses the duplicate name at write time anyway, and failing before the writer
- names the column instead of a qualified field in a schema error.
+ Names are compared the way the load writes them, after snake_case normalization, so
+ `_PH_CDC_SEQ` collides with `_ph_cdc_seq`. Capture raises it before it writes the buffer: buffer
+ files derive their name, ordering, and retry cleanup from the engine position, and ordering or
+ cleaning up by customer data can silently delete unconsumed buffer files.
+
+ The SCD2 stamp raises it too, because Delta refuses the duplicate name at write time anyway, and
+ failing before the writer names the column instead of a qualified field in a schema error.
"""
@@ -174,8 +173,10 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {
False,
),
CDCErrorCategory.RESERVED_COLUMN_CONFLICT: (
- "A source table has a column named _ph_cdc_seq, which PostHog uses for change data capture. "
- "Rename the column on your database, or choose another sync method for that table.",
+ "A source table has a column that loads under a name PostHog writes for change data capture, "
+ "such as _ph_cdc_seq or _ph_deleted, or valid_from or valid_to on a table with a history table. "
+ "Rename the column on your database, leave it out of the synced columns, or choose another sync "
+ "method for that table.",
False,
),
CDCErrorCategory.UNKNOWN: (
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py
index c67f6f719c9f..5a11203067b5 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py
@@ -56,9 +56,9 @@ def _pk_tuples(table: pa.Table, primary_keys: list[str]) -> list[tuple]:
def has_engine_seq(table: pa.Table) -> bool:
"""Whether the position column is ours.
- A source table may have its own `_ph_cdc_seq`, which the batcher passes through untouched, so
- the name proves nothing. Trusting a source value would let it poison the position and have its
- own later rows dropped.
+ A snapshot load, or a table loaded before capture refused the name, can hold a source column
+ called `_ph_cdc_seq`, so the name proves nothing. Trusting a source value would let it poison the
+ position and have its own later rows dropped.
"""
if CDC_SEQ_COLUMN not in table.column_names:
return False
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
index ce94cc745109..1a11ed1f6cab 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
@@ -22,8 +22,9 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
build_scd2_table,
enrich_delete_rows,
enrich_toast_omitted_rows,
- history_column_collisions,
is_history_table_mode,
+ reserved_column_collisions,
+ reserved_column_names,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnError
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
@@ -615,19 +616,17 @@ class TestSeqColumn:
assert CDC_SEQ_COLUMN not in table.column_names
- def test_source_column_named_like_seq_wins_and_append_is_skipped(self):
- # A user column literally named _ph_cdc_seq must pass through with its own
- # values; the engine seq append is skipped rather than shadowing it.
+ @parameterized.expand([(CDC_SEQ_COLUMN,), ("_PH_CDC_SEQ",), ("_phDeletedAt",)])
+ def test_a_source_column_that_loads_like_a_capture_column_is_refused(self, column: str) -> None:
+ # The load renames `_PH_CDC_SEQ` to `_ph_cdc_seq`, so the snapshot would put customer values
+ # in the column every position decision reads.
batcher = ChangeEventBatcher(position_to_seq=_pg_position_to_seq)
- batcher.add(_make_event(op="I", position="0/100", columns={"id": 1, CDC_SEQ_COLUMN: 42}))
- table = batcher.flush()["users"]
+ batcher.add(_make_event(op="I", position="0/100", columns={"id": 1, column: 42}))
+
+ tables = batcher.flush()
- assert table.column_names.count(CDC_SEQ_COLUMN) == 1
- assert table.column(CDC_SEQ_COLUMN).to_pylist() == [42]
- # Not last: the caller's strip-by-last-field rule must not remove it.
- assert table.schema.field(table.num_columns - 1).name != CDC_SEQ_COLUMN
- # Unstamped, so the load side cannot mistake these user values for engine positions.
- assert table.schema.field(CDC_SEQ_COLUMN).metadata is None
+ assert "users" not in tables
+ assert column in str(batcher.reserved_collisions["users"])
def test_engine_seq_stamp_survives_the_whole_transform_chain(self):
# The load side gates every position decision on this stamp, and each transform below
@@ -713,24 +712,36 @@ class TestScd2TimestampType:
assert result.column(SCD2_VALID_TO_COLUMN).to_pylist()[0] is not None
-class TestHistoryColumnCollisions:
+class TestReservedColumnCollisions:
@parameterized.expand(
[
- ("exact_name", ["id", "valid_to"], None, None, ["valid_to"]),
- ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], None, None, ["VALID_TO", "validFrom"]),
- ("left_out_of_enabled_columns", ["id", "validFrom"], ["id"], None, []),
- ("kept_as_the_incremental_field", ["id", "valid_from"], [], "valid_from", ["valid_from"]),
+ ("exact_name", ["id", "valid_to"], True, None, None, ["valid_to"]),
+ ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], True, None, None, ["VALID_TO", "validFrom"]),
+ ("capture_name_the_load_renames", ["id", "_PH_CDC_SEQ"], False, None, None, ["_PH_CDC_SEQ"]),
+ ("validity_name_without_history", ["id", "valid_from"], False, None, None, []),
+ ("left_out_of_enabled_columns", ["id", "validFrom"], True, ["id"], None, []),
+ ("kept_as_the_incremental_field", ["id", "valid_from"], True, [], "valid_from", ["valid_from"]),
]
)
def test_names_the_source_columns_that_collide(
self,
_name: str,
column_names: list[str],
+ history: bool,
enabled_columns: list[str] | None,
incremental_field: str | None,
expected: list[str],
) -> None:
- assert history_column_collisions(column_names, enabled_columns, ["id"], incremental_field) == expected
+ assert (
+ reserved_column_collisions(
+ column_names,
+ reserved_column_names(history=history),
+ enabled_columns=enabled_columns,
+ primary_keys=["id"],
+ incremental_field=incremental_field,
+ )
+ == expected
+ )
class TestIsHistoryTableMode:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
index f17d1f99aa55..10d20de3173c 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
@@ -36,7 +36,11 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
CDC_SEQ_PROVENANCE,
ChangeEventBatcher,
)
-from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCErrorCategory, cdc_error_info
+from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import (
+ CDCErrorCategory,
+ CDCReservedColumnError,
+ cdc_error_info,
+)
from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import (
cancel_running_sync,
stage_handed_over_reset,
@@ -2315,21 +2319,53 @@ class TestBufferedIngressCapture:
team_id=schema.team_id, schema_id=str(schema.id), restart_seq=0x100
)
- def test_a_source_column_named_like_seq_fails_the_buffered_run(self):
- # The batcher skips its append on the collision, so the file's name, ordering, and retry
- # cleanup would all derive from customer data — cleanup can then delete unconsumed files.
+ @parameterized.expand([("table_writes_the_buffer", False, True), ("table_awaiting_reset", True, False)])
+ def test_a_capture_column_collision_fails_only_a_table_that_writes_the_buffer(
+ self, _name: str, awaiting_reset: bool, refused: bool
+ ) -> None:
+ # A reset discards the table's queued changes, so its collision must not stop the other tables.
source = _make_source()
- schema = _make_schema("users", cdc_mode="streaming", source=source)
- events = [_make_event(op="I", position="0/100", columns={"id": 1, CDC_SEQ_COLUMN: 42})]
+ act = _make_extract_activity(source)
+ act.schema_by_name = {"users": _make_schema("users", source=source)}
+ act._buffered_table_names = {"users"}
+ if awaiting_reset:
+ act._tables_awaiting_reset.add("users")
+ act.batcher = ChangeEventBatcher(position_to_seq=lambda position: 1)
+ act.batcher.add(_make_event(op="I", columns={"id": 1, "_PH_DELETED": True}))
+ tables = act.batcher.flush()
+
+ if refused:
+ with pytest.raises(CDCReservedColumnError, match="users"):
+ act._process_flush(tables)
+ else:
+ act._process_flush(tables)
- with (
- _capture_harness(source, [schema], events) as capture,
- pytest.raises(NonRetryableException, match=CDC_SEQ_COLUMN),
- ):
- capture.extract()
+ @parameterized.expand(
+ [
+ ("position_column", CDC_SEQ_COLUMN, "consolidated", True),
+ ("position_column_the_load_renames", "_PH_CDC_SEQ", "consolidated", True),
+ ("validity_column_with_a_history_table", "validFrom", "both", True),
+ ("validity_column_without_a_history_table", "valid_from", "consolidated", False),
+ ]
+ )
+ def test_a_source_column_named_like_a_written_column_fails_the_buffered_run(
+ self, _name: str, column: str, cdc_table_mode: str, refused: bool
+ ) -> None:
+ # Written anyway, the file's name, ordering, and retry cleanup could derive from customer
+ # data, and cleanup can then delete unconsumed files.
+ source = _make_source()
+ schema = _make_schema("users", cdc_mode="streaming", cdc_table_mode=cdc_table_mode, source=source)
+ events = [_make_event(op="I", position="0/100", columns={"id": 1, column: 42})]
- capture.buffer.write_batch.assert_not_called()
- capture.reader.confirm_position.assert_not_called()
+ with _capture_harness(source, [schema], events) as capture:
+ if refused:
+ with pytest.raises(NonRetryableException, match="change data capture"):
+ capture.extract()
+ else:
+ capture.extract()
+
+ assert capture.buffer.write_batch.called is not refused
+ assert capture.reader.confirm_position.called is not refused
@parameterized.expand(
[
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index 7b8059876bd5..d2620a3360b8 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -1341,6 +1341,18 @@ class TestExternalDataSchema(APIBaseTest):
None,
"_ph_cdc_seq",
),
+ (
+ "rejects_a_source_column_the_load_renames_to_a_capture_column",
+ {
+ "primary_key_columns": ["id"],
+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "_PH_CDC_SEQ"}]},
+ },
+ None,
+ status.HTTP_400_BAD_REQUEST,
+ ExternalDataSchema.SyncType.FULL_REFRESH,
+ None,
+ "_ph_cdc_seq",
+ ),
(
"rejects_a_history_table_for_a_source_column_named_like_a_validity_column",
{
@@ -4587,6 +4599,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
[
(
"turning_history_on_over_a_synced_column_is_refused",
+ "valid_to",
"consolidated",
None,
{"cdc_table_mode": "both"},
@@ -4594,6 +4607,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
),
(
"turning_history_on_over_a_column_left_out_is_allowed",
+ "valid_to",
"consolidated",
["id"],
{"cdc_table_mode": "both"},
@@ -4601,17 +4615,43 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
),
(
"adding_the_column_to_enabled_columns_is_refused",
+ "valid_to",
"both",
["id"],
{"enabled_columns": ["id", "valid_to"]},
True,
),
- ("editing_a_table_that_already_syncs_it_is_allowed", "both", None, {"enabled_columns": None}, False),
+ (
+ "editing_a_table_that_already_syncs_it_is_allowed",
+ "valid_to",
+ "both",
+ None,
+ {"enabled_columns": None},
+ False,
+ ),
+ # Capture columns are refused on every CDC table, history or not.
+ (
+ "adding_a_capture_column_to_enabled_columns_is_refused",
+ "_PH_DELETED",
+ "consolidated",
+ ["id"],
+ {"enabled_columns": ["id", "_PH_DELETED"]},
+ True,
+ ),
+ (
+ "editing_a_table_that_already_syncs_a_capture_column_is_allowed",
+ "_PH_DELETED",
+ "consolidated",
+ None,
+ {"enabled_columns": None},
+ False,
+ ),
]
)
- def test_patch_checks_the_validity_columns_the_history_table_receives(
+ def test_patch_checks_the_columns_a_cdc_sync_writes(
self,
_name: str,
+ column: str,
initial_mode: str,
initial_enabled_columns: list[str] | None,
request_body: dict,
@@ -4622,7 +4662,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
)
self.schema.sync_type_config["cdc_table_mode"] = initial_mode
- self.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": "valid_to"}]}
+ self.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": column}]}
self.schema.enabled_columns = initial_enabled_columns
self.schema.save()
@@ -4638,7 +4678,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
):
serializer.is_valid(raise_exception=True)
if refused:
- with pytest.raises(ValidationError, match="valid_to"):
+ with pytest.raises(ValidationError, match=column):
serializer.save()
else:
serializer.save()
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 883552c83881..56477cc0e485 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -5036,6 +5036,14 @@ class TestExternalDataSource(APIBaseTest):
"consolidated",
None,
),
+ (
+ "capture_column_the_load_renames",
+ [("id", "uuid", False), ("_PH_DELETED", "boolean", True)],
+ {"tracking_link": ["id"]},
+ "tracking_link (_ph_deleted)",
+ "consolidated",
+ None,
+ ),
(
"validity_column_with_a_history_table",
[("id", "uuid", False), ("valid_to", "timestamp", True)],