:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): refuse every column name a cdc sync writes (#123543)
Daniel RC7ac6bc808335
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.pyindex 3e36a91f6b1d..2ca5d6b15764 100644--- a/products/warehouse_sources/backend/facade/source_management.py+++ b/products/warehouse_sources/backend/facade/source_management.py@@ -29,9 +29,9 @@ _LAZY = {"SELF_MANAGED_LAG_REASON": "cdc.broken","tables_wait_for_repair": "cdc.broken","purge_buffer_prefix": "cdc.buffer",- "CDC_SEQ_COLUMN": "cdc.batcher","is_history_table_mode": "cdc.batcher",- "history_column_collisions": "cdc.batcher",+ "reserved_column_collisions": "cdc.batcher",+ "reserved_column_names": "cdc.batcher","BUFFER_LANE": "cdc.snapshot_lane","CDC_RESET_PENDING_KEY": "cdc.snapshot_lane","hand_reset_to_capture_if_sync_running": "cdc.snapshot_lane",diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex 2a3a82d170f4..067527870a35 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -66,7 +66,6 @@ from products.warehouse_sources.backend.facade.source_inputs import SourceInputsfrom products.warehouse_sources.backend.facade.source_management import (BUFFER_LANE,CDC_RESET_PENDING_KEY,- CDC_SEQ_COLUMN,AnySource,RowFilterValidationError,SourceRegistry,@@ -75,10 +74,11 @@ from products.warehouse_sources.backend.facade.source_management import (filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,get_cdc_adapter,hand_reset_to_capture_if_sync_running,- history_column_collisions,is_history_table_mode,purge_buffer_prefix,repair_is_running,+ reserved_column_collisions,+ reserved_column_names,resnapshot_stays_in_buffer,source_type_supports_cdc,tables_wait_for_repair,@@ -457,17 +457,7 @@ def _source_column_names(instance: ExternalDataSchema) -> set[str]:return {column["name"] for column in columns if isinstance(column, dict) and isinstance(column.get("name"), str)}-def _refuse_reserved_cdc_column(instance: ExternalDataSchema) -> None:- """Capture stamps each change with this column, so a source column of the same name would fail- the source's sync."""- if CDC_SEQ_COLUMN in _source_column_names(instance):- raise ValidationError(- "Change data capture can't sync a column named _ph_cdc_seq, because PostHog uses that name. "- "Rename the column on your database, or choose another sync method for this table."- )---def _refuse_reserved_history_columns(+def _refuse_reserved_columns(instance: ExternalDataSchema,*,previous_mode: str | None,@@ -477,23 +467,31 @@ def _refuse_reserved_history_columns(new_enabled_columns: list[str] | None,new_primary_keys: list[str] | None,) -> None:- """The history table stamps each row version with `valid_from` and `valid_to`, so a synced source- column of the same name fails every load into it. Only a collision this request adds is refused,- so a table that already has one can still be edited or switched back."""- if not is_history_table_mode(new_mode):- return+ """Capture and the history table write their own columns next to the source's, so a synced source+ column that loads under one of those names fails the sync. Only a collision this request adds is+ refused, so a table that already has one can still be edited or switched back."""columns = _source_column_names(instance)- taken = history_column_collisions(columns, new_enabled_columns, new_primary_keys, instance.incremental_field)- if instance.sync_type == ExternalDataSchema.SyncType.CDC and is_history_table_mode(previous_mode):- existing = history_column_collisions(- columns, previous_enabled_columns, previous_primary_keys, instance.incremental_field+ taken = reserved_column_collisions(+ columns,+ reserved_column_names(history=is_history_table_mode(new_mode)),+ enabled_columns=new_enabled_columns,+ primary_keys=new_primary_keys,+ incremental_field=instance.incremental_field,+ )+ if taken and instance.sync_type == ExternalDataSchema.SyncType.CDC:+ existing = reserved_column_collisions(+ columns,+ reserved_column_names(history=is_history_table_mode(previous_mode)),+ enabled_columns=previous_enabled_columns,+ primary_keys=previous_primary_keys,+ incremental_field=instance.incremental_field,)taken = [column for column in taken if column not in existing]if taken:raise ValidationError(- f"A history table can't store the column {' or '.join(taken)}, because PostHog uses valid_from and "- "valid_to to record when each version was current. Rename the column on your database, or choose "- "the consolidated table only for this table."+ f"Change data capture can't sync the column {' or '.join(taken)}, because it loads under a name "+ "PostHog writes. Rename the column on your database, leave it out of the synced columns, or change "+ "this table's sync method or output tables.")@@ -1331,7 +1329,6 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers# discovery already stored; refuse the switch when neither is set._apply_primary_key_columns(data, payload, instance, "CDC")if instance.sync_type != ExternalDataSchema.SyncType.CDC:- _refuse_reserved_cdc_column(instance)# A table joining CDC starts from a fresh snapshot, even when an earlier sync# method or an earlier CDC period already loaded it. The snapshot moves to# streaming only when initial_sync_complete goes from False to True, so a table@@ -1369,7 +1366,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers# can also bring a validity column into the history table.if (sync_type if "sync_type" in data else instance.sync_type) == ExternalDataSchema.SyncType.CDC:resulting_config = validated_data.get("sync_type_config", instance.sync_type_config) or {}- _refuse_reserved_history_columns(+ _refuse_reserved_columns(instance,previous_mode=previous_cdc_table_mode,previous_enabled_columns=instance.enabled_columns,diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.pyindex f360c4657e69..7869fab3340b 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py@@ -51,7 +51,6 @@ from products.warehouse_sources.backend.facade.models import ()from products.warehouse_sources.backend.facade.source_inputs import SourceInputsStore, UnreadableSourceInputsErrorfrom products.warehouse_sources.backend.facade.source_management import (- CDC_SEQ_COLUMN,PREVIEW_DEFAULT_ROWS,PREVIEW_MAX_ROWS,AnySource,@@ -72,10 +71,11 @@ from products.warehouse_sources.backend.facade.source_management import (fetch_docs_text,filter_dwh_columns_by_enabled_columns,get_cdc_adapter,- history_column_collisions,is_history_table_mode,is_team_allowlisted_for_internal_hosts,new_source_requires_ssl,+ reserved_column_collisions,+ reserved_column_names,sql_schema_metadata,validate_and_coerce_row_filters,)@@ -1454,18 +1454,23 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):},)- # Capture stamps each change with this column, so a source column of the same name would- # fail the source's first sync. Refuse before any replication state exists.+ # Capture and the history table write their own columns next to the source's, so a synced+ # source column that loads under one of those names would fail the source's first sync.+ # Refuse before any replication state exists.+ reserved_columns_by_table = {+ schema["name"]: reserved_column_collisions(+ [column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],+ reserved_column_names(history=is_history_table_mode(schema.get("cdc_table_mode"))),+ enabled_columns=_payload_enabled_columns(schema),+ primary_keys=pk_columns_by_table.get(schema["name"]),+ )+ for schema in payload_schemas+ if schema.get("sync_type") == "cdc"+ and schema.get("should_sync", False)+ and isinstance(schema.get("name"), str)+ }tables_with_reserved_column = sorted(- {- schema["name"]- for schema in payload_schemas- if schema.get("sync_type") == "cdc"- and schema.get("should_sync", False)- and isinstance(schema.get("name"), str)- and CDC_SEQ_COLUMN- in {column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])}- }+ f"{table} ({', '.join(columns)})" for table, columns in reserved_columns_by_table.items() if columns)if tables_with_reserved_column:new_source_model.delete()@@ -1473,40 +1478,9 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):status=status.HTTP_400_BAD_REQUEST,data={"message": (- "Change data capture can't sync a column named _ph_cdc_seq, because PostHog uses "- "that name. Rename the column on your database, or choose another sync method for "- f"these tables: {', '.join(tables_with_reserved_column)}."- )- },- )-- # The history table stamps each row version with these columns, so a synced source column- # of the same name would fail every load into it.- tables_with_validity_column = sorted(- {- schema["name"]- for schema in payload_schemas- if schema.get("sync_type") == "cdc"- and schema.get("should_sync", False)- and is_history_table_mode(schema.get("cdc_table_mode"))- and isinstance(schema.get("name"), str)- and history_column_collisions(- [column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],- _payload_enabled_columns(schema),- pk_columns_by_table.get(schema["name"]),- )- }- )- if tables_with_validity_column:- new_source_model.delete()- return Response(- status=status.HTTP_400_BAD_REQUEST,- data={- "message": (- "A history table can't store a column that loads as valid_from or valid_to, because "- "PostHog uses those names to record when each version was current. Rename the column on "- "your database, or choose the consolidated table only for these tables: "- f"{', '.join(tables_with_validity_column)}."+ "Change data capture can't sync a column that loads under a name PostHog writes. Rename "+ "the column on your database, leave it out of the synced columns, or change the sync "+ f"method or output tables for these tables: {'; '.join(tables_with_reserved_column)}.")},)diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.pyindex 7badfcdd00ee..d79c5e7e0d08 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py@@ -46,9 +46,12 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters impor)from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_SEQ_COLUMN,+ HISTORY_COLUMNS,ChangeEventBatcher,enrich_delete_rows,enrich_toast_omitted_rows,+ is_history_table_mode,+ reserved_column_collisions,)from products.warehouse_sources.backend.temporal.data_imports.cdc.billing_expiry import (blocked_past_buffer_retention,@@ -350,17 +353,25 @@ class CDCExtractActivity:# ------------------------------------------------------------------# Per-flush processing# ------------------------------------------------------------------+ def _writes_buffer_for(self, table_name: str) -> bool:+ return (+ table_name in self.schema_by_name+ and table_name in self._buffered_table_names+ and table_name not in self._tables_awaiting_reset+ )+def _process_flush(self, tables: dict[str, pa.Table]) -> None:"""Enrich one micro-batch and write each captured table's changes to the S3 change buffer."""events_extracted = 0+ assert self.batcher is not None+ for table_name, error in self.batcher.reserved_collisions.items():+ if self._writes_buffer_for(table_name):+ raise error+for table_name, raw_table in tables.items():schema = self.schema_by_name.get(table_name)- if (- schema is None- or table_name not in self._buffered_table_names- or table_name in self._tables_awaiting_reset- ):+ if schema is None or not self._writes_buffer_for(table_name):continueself._safe_heartbeat()@@ -368,6 +379,13 @@ class CDCExtractActivity:# raw_table has one row per source change event.events_extracted += raw_table.num_rows+ # The batcher already refused the capture columns. The history table's own columns are+ # checked here, because only the schema says whether this table keeps history.+ if is_history_table_mode(schema.cdc_table_mode):+ taken = reserved_column_collisions(raw_table.column_names, HISTORY_COLUMNS)+ if taken:+ raise CDCReservedColumnError(f"Table {table_name} has history table column(s) {taken}")+key_columns = self.pk_columns_by_table.get(table_name, [])# TOAST fill first so DELETE enrichment copies resolved values, not the@@ -377,11 +395,10 @@ class CDCExtractActivity:if not enriched_table.num_rows:continue- # A source column named _ph_cdc_seq means the batcher could not append the- # engine position; writing anyway would name, order, and clean up files by- # customer data — cleanup can then delete unconsumed files (see errors.py).+ # The batcher refuses a source column of this name, so only a capture bug gets here. Writing+ # without the engine position could make cleanup delete unconsumed files (see errors.py).if not has_engine_seq(enriched_table):- raise CDCReservedColumnError(f"Table {table_name} has a source column named {CDC_SEQ_COLUMN}")+ raise RuntimeError(f"Table {table_name} reached the buffer without the engine position")self._write_buffer_file(schema, table_name, enriched_table)if events_extracted:diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.pyindex 278a583313ad..2646fe7f0d96 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py@@ -21,7 +21,7 @@ DELETED_AT_COLUMN = "_ph_deleted_at"# and (in Phase B) drives the merge's monotonicity guard.CDC_SEQ_COLUMN = "_ph_cdc_seq"# Marks CDC_SEQ_COLUMN as engine-produced, so readers can tell it from a source column of the same-# name (see the collision skip in _events_to_table).+# name.CDC_SEQ_PROVENANCE = {b"posthog_cdc": b"engine_position"}# Suffix of the SCD2 companion table's resource name ({schema.name}_cdc).@@ -57,22 +57,36 @@ def is_history_table_mode(cdc_table_mode: object) -> bool:return isinstance(cdc_table_mode, str) and cdc_table_mode in CDC_HISTORY_TABLE_MODES-def history_column_collisions(+# Every column capture stamps onto a change row. The consolidated table and the history table both store them.+CAPTURE_COLUMNS: frozenset[str] = frozenset(+ {CDC_OP_COLUMN, CDC_TIMESTAMP_COLUMN, CDC_SEQ_COLUMN, DELETED_COLUMN, DELETED_AT_COLUMN, TOAST_OMITTED_COLUMN}+)+# The columns that only the history table adds.+HISTORY_COLUMNS: frozenset[str] = frozenset({SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN})+++def reserved_column_names(*, history: bool) -> frozenset[str]:+ """Every name a CDC sync writes next to the source's own columns."""+ return CAPTURE_COLUMNS | HISTORY_COLUMNS if history else CAPTURE_COLUMNS+++def reserved_column_collisions(column_names: Iterable[str],- enabled_columns: list[str] | None,- primary_keys: list[str] | None,+ reserved: frozenset[str],+ *,+ enabled_columns: list[str] | None = None,+ primary_keys: list[str] | None = None,incremental_field: str | None = None,) -> list[str]:- """Source columns that the history table would receive under its validity column names."""+ """Source columns that would load under a name in `reserved`."""synced = set(column_names)# The same projection capture applies (`filter_columns_by_enabled_columns`), kept inline because# that module's package imports Django models and this one sits on the capture path.if enabled_columns is not None:synced &= {*enabled_columns, *(primary_keys or []), *([incremental_field] if incremental_field else [])}- # The load renames columns to snake_case before the history transform adds the validity columns,- # so `validFrom` collides too. The source names are returned, because those are what a user renames.- validity_columns = {SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN}- return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in validity_columns)+ # The load renames columns to snake_case, so `validFrom` and `_PH_CDC_SEQ` collide too. The source+ # names are returned, because those are what a user renames.+ return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in reserved)# Columns that are CDC/SCD2 metadata — always filled from the event itself,@@ -124,6 +138,10 @@ class ChangeEventBatcher:# Engine adapter's position_serialized → int converter. When set, flushed# tables carry CDC_SEQ_COLUMN; when None the column is absent entirely.self._position_to_seq = position_to_seq+ # Tables the last flush could not build, because a source column collides with a capture column.+ # The caller raises for them only once it knows the table is not waiting for a reset, because a+ # reset discards the table's changes.+ self.reserved_collisions: dict[str, CDCReservedColumnError] = {}def add(self, event: ChangeEvent) -> None:size = self._estimate_event_bytes(event)@@ -142,11 +160,15 @@ class ChangeEventBatcher:Returns empty dict if no events buffered."""result: dict[str, pa.Table] = {}+ self.reserved_collisions = {}for table_name, events in self._events.items():if not events:continue- result[table_name] = _events_to_table(events, position_to_seq=self._position_to_seq)+ try:+ result[table_name] = _events_to_table(events, position_to_seq=self._position_to_seq)+ except CDCReservedColumnError as error:+ self.reserved_collisions[table_name] = errorself._events.clear()self._table_bytes.clear()@@ -453,13 +475,11 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:A two-step merge + append in the load processor closes previous "current"rows (sets valid_to) when a new batch is written for the same PK."""- taken = {SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN} & set(pa_table.column_names)+ taken = reserved_column_collisions(pa_table.column_names, HISTORY_COLUMNS)if taken:# Delta refuses a duplicate column name at write time; failing here names the column and# keeps a half-built batch out of the writer.- raise CDCReservedColumnError(- f"Source column(s) {sorted(taken)} collide with the history table's validity columns"- )+ raise CDCReservedColumnError(f"Source column(s) {taken} collide with the history table's validity columns")# Taken from the batch rather than assumed: `valid_from` is the timestamp column's own values,# and declaring a type it does not have makes pyarrow reject the append outright.ts_type = (@@ -517,6 +537,12 @@ def _events_to_table(events: list[ChangeEvent], position_to_seq: Callable[[str],for col_name in event.omitted_columns:all_columns[col_name] = Nonecolumn_names = list(all_columns.keys())+ taken = reserved_column_collisions(column_names, CAPTURE_COLUMNS)+ if taken:+ raise CDCReservedColumnError(+ f"Table {events[0].table_name} has source column(s) {taken} that collide with the columns change data "+ "capture writes"+ )# Authoritative Arrow type per column from the source schema (same for every event# of a relation). Lets all-null micro-batches keep the column's real type.@@ -583,9 +609,7 @@ def _events_to_table(events: list[ChangeEvent], position_to_seq: Callable[[str],arrays.append(pa.array(omitted_lists, type=pa.list_(pa.string())))fields.append(pa.field(TOAST_OMITTED_COLUMN, pa.list_(pa.string())))- # Skip on collision: a source column literally named _ph_cdc_seq is never overwritten. The batch- # then carries no engine seq, and capture refuses to write it to the buffer.- if position_to_seq is not None and CDC_SEQ_COLUMN not in column_names:+ if position_to_seq is not None:seq_values = [position_to_seq(event.position_serialized) for event in events]arrays.append(pa.array(seq_values, type=pa.int64()))fields.append(pa.field(CDC_SEQ_COLUMN, pa.int64(), metadata=CDC_SEQ_PROVENANCE))diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.pyindex 291222b0d500..793fce39f5d3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py@@ -74,14 +74,13 @@ class CDCReservedColumnError(Exception):Non-retryable: the collision is a property of the customer's table, so replaying re-fails.- For the engine position column it is raised when capture writes the buffer — buffer files- derive their name, ordering, and retry cleanup from it, and a same-named source column means- the batcher could not append it. Writing anyway would order and clean up by customer data,- which can silently delete unconsumed buffer files.-- For the history table's validity columns it is raised on every path, from the SCD2 stamp- itself: Delta refuses the duplicate name at write time anyway, and failing before the writer- names the column instead of a qualified field in a schema error.+ Names are compared the way the load writes them, after snake_case normalization, so+ `_PH_CDC_SEQ` collides with `_ph_cdc_seq`. Capture raises it before it writes the buffer: buffer+ files derive their name, ordering, and retry cleanup from the engine position, and ordering or+ cleaning up by customer data can silently delete unconsumed buffer files.++ The SCD2 stamp raises it too, because Delta refuses the duplicate name at write time anyway, and+ failing before the writer names the column instead of a qualified field in a schema error."""@@ -174,8 +173,10 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {False,),CDCErrorCategory.RESERVED_COLUMN_CONFLICT: (- "A source table has a column named _ph_cdc_seq, which PostHog uses for change data capture. "- "Rename the column on your database, or choose another sync method for that table.",+ "A source table has a column that loads under a name PostHog writes for change data capture, "+ "such as _ph_cdc_seq or _ph_deleted, or valid_from or valid_to on a table with a history table. "+ "Rename the column on your database, leave it out of the synced columns, or choose another sync "+ "method for that table.",False,),CDCErrorCategory.UNKNOWN: (diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.pyindex c67f6f719c9f..5a11203067b5 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py@@ -56,9 +56,9 @@ def _pk_tuples(table: pa.Table, primary_keys: list[str]) -> list[tuple]:def has_engine_seq(table: pa.Table) -> bool:"""Whether the position column is ours.- A source table may have its own `_ph_cdc_seq`, which the batcher passes through untouched, so- the name proves nothing. Trusting a source value would let it poison the position and have its- own later rows dropped.+ A snapshot load, or a table loaded before capture refused the name, can hold a source column+ called `_ph_cdc_seq`, so the name proves nothing. Trusting a source value would let it poison the+ position and have its own later rows dropped."""if CDC_SEQ_COLUMN not in table.column_names:return Falsediff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.pyindex ce94cc745109..1a11ed1f6cab 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py@@ -22,8 +22,9 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importbuild_scd2_table,enrich_delete_rows,enrich_toast_omitted_rows,- history_column_collisions,is_history_table_mode,+ reserved_column_collisions,+ reserved_column_names,)from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnErrorfrom products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent@@ -615,19 +616,17 @@ class TestSeqColumn:assert CDC_SEQ_COLUMN not in table.column_names- def test_source_column_named_like_seq_wins_and_append_is_skipped(self):- # A user column literally named _ph_cdc_seq must pass through with its own- # values; the engine seq append is skipped rather than shadowing it.+ @parameterized.expand([(CDC_SEQ_COLUMN,), ("_PH_CDC_SEQ",), ("_phDeletedAt",)])+ def test_a_source_column_that_loads_like_a_capture_column_is_refused(self, column: str) -> None:+ # The load renames `_PH_CDC_SEQ` to `_ph_cdc_seq`, so the snapshot would put customer values+ # in the column every position decision reads.batcher = ChangeEventBatcher(position_to_seq=_pg_position_to_seq)- batcher.add(_make_event(op="I", position="0/100", columns={"id": 1, CDC_SEQ_COLUMN: 42}))- table = batcher.flush()["users"]+ batcher.add(_make_event(op="I", position="0/100", columns={"id": 1, column: 42}))++ tables = batcher.flush()- assert table.column_names.count(CDC_SEQ_COLUMN) == 1- assert table.column(CDC_SEQ_COLUMN).to_pylist() == [42]- # Not last: the caller's strip-by-last-field rule must not remove it.- assert table.schema.field(table.num_columns - 1).name != CDC_SEQ_COLUMN- # Unstamped, so the load side cannot mistake these user values for engine positions.- assert table.schema.field(CDC_SEQ_COLUMN).metadata is None+ assert "users" not in tables+ assert column in str(batcher.reserved_collisions["users"])def test_engine_seq_stamp_survives_the_whole_transform_chain(self):# The load side gates every position decision on this stamp, and each transform below@@ -713,24 +712,36 @@ class TestScd2TimestampType:assert result.column(SCD2_VALID_TO_COLUMN).to_pylist()[0] is not None-class TestHistoryColumnCollisions:+class TestReservedColumnCollisions:@parameterized.expand([- ("exact_name", ["id", "valid_to"], None, None, ["valid_to"]),- ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], None, None, ["VALID_TO", "validFrom"]),- ("left_out_of_enabled_columns", ["id", "validFrom"], ["id"], None, []),- ("kept_as_the_incremental_field", ["id", "valid_from"], [], "valid_from", ["valid_from"]),+ ("exact_name", ["id", "valid_to"], True, None, None, ["valid_to"]),+ ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], True, None, None, ["VALID_TO", "validFrom"]),+ ("capture_name_the_load_renames", ["id", "_PH_CDC_SEQ"], False, None, None, ["_PH_CDC_SEQ"]),+ ("validity_name_without_history", ["id", "valid_from"], False, None, None, []),+ ("left_out_of_enabled_columns", ["id", "validFrom"], True, ["id"], None, []),+ ("kept_as_the_incremental_field", ["id", "valid_from"], True, [], "valid_from", ["valid_from"]),])def test_names_the_source_columns_that_collide(self,_name: str,column_names: list[str],+ history: bool,enabled_columns: list[str] | None,incremental_field: str | None,expected: list[str],) -> None:- assert history_column_collisions(column_names, enabled_columns, ["id"], incremental_field) == expected+ assert (+ reserved_column_collisions(+ column_names,+ reserved_column_names(history=history),+ enabled_columns=enabled_columns,+ primary_keys=["id"],+ incremental_field=incremental_field,+ )+ == expected+ )class TestIsHistoryTableMode:diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.pyindex f17d1f99aa55..10d20de3173c 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py@@ -36,7 +36,11 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importCDC_SEQ_PROVENANCE,ChangeEventBatcher,)-from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCErrorCategory, cdc_error_info+from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import (+ CDCErrorCategory,+ CDCReservedColumnError,+ cdc_error_info,+)from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import (cancel_running_sync,stage_handed_over_reset,@@ -2315,21 +2319,53 @@ class TestBufferedIngressCapture:team_id=schema.team_id, schema_id=str(schema.id), restart_seq=0x100)- def test_a_source_column_named_like_seq_fails_the_buffered_run(self):- # The batcher skips its append on the collision, so the file's name, ordering, and retry- # cleanup would all derive from customer data — cleanup can then delete unconsumed files.+ @parameterized.expand([("table_writes_the_buffer", False, True), ("table_awaiting_reset", True, False)])+ def test_a_capture_column_collision_fails_only_a_table_that_writes_the_buffer(+ self, _name: str, awaiting_reset: bool, refused: bool+ ) -> None:+ # A reset discards the table's queued changes, so its collision must not stop the other tables.source = _make_source()- schema = _make_schema("users", cdc_mode="streaming", source=source)- events = [_make_event(op="I", position="0/100", columns={"id": 1, CDC_SEQ_COLUMN: 42})]+ act = _make_extract_activity(source)+ act.schema_by_name = {"users": _make_schema("users", source=source)}+ act._buffered_table_names = {"users"}+ if awaiting_reset:+ act._tables_awaiting_reset.add("users")+ act.batcher = ChangeEventBatcher(position_to_seq=lambda position: 1)+ act.batcher.add(_make_event(op="I", columns={"id": 1, "_PH_DELETED": True}))+ tables = act.batcher.flush()++ if refused:+ with pytest.raises(CDCReservedColumnError, match="users"):+ act._process_flush(tables)+ else:+ act._process_flush(tables)- with (- _capture_harness(source, [schema], events) as capture,- pytest.raises(NonRetryableException, match=CDC_SEQ_COLUMN),- ):- capture.extract()+ @parameterized.expand(+ [+ ("position_column", CDC_SEQ_COLUMN, "consolidated", True),+ ("position_column_the_load_renames", "_PH_CDC_SEQ", "consolidated", True),+ ("validity_column_with_a_history_table", "validFrom", "both", True),+ ("validity_column_without_a_history_table", "valid_from", "consolidated", False),+ ]+ )+ def test_a_source_column_named_like_a_written_column_fails_the_buffered_run(+ self, _name: str, column: str, cdc_table_mode: str, refused: bool+ ) -> None:+ # Written anyway, the file's name, ordering, and retry cleanup could derive from customer+ # data, and cleanup can then delete unconsumed files.+ source = _make_source()+ schema = _make_schema("users", cdc_mode="streaming", cdc_table_mode=cdc_table_mode, source=source)+ events = [_make_event(op="I", position="0/100", columns={"id": 1, column: 42})]- capture.buffer.write_batch.assert_not_called()- capture.reader.confirm_position.assert_not_called()+ with _capture_harness(source, [schema], events) as capture:+ if refused:+ with pytest.raises(NonRetryableException, match="change data capture"):+ capture.extract()+ else:+ capture.extract()++ assert capture.buffer.write_batch.called is not refused+ assert capture.reader.confirm_position.called is not refused@parameterized.expand([diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex 7b8059876bd5..d2620a3360b8 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -1341,6 +1341,18 @@ class TestExternalDataSchema(APIBaseTest):None,"_ph_cdc_seq",),+ (+ "rejects_a_source_column_the_load_renames_to_a_capture_column",+ {+ "primary_key_columns": ["id"],+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "_PH_CDC_SEQ"}]},+ },+ None,+ status.HTTP_400_BAD_REQUEST,+ ExternalDataSchema.SyncType.FULL_REFRESH,+ None,+ "_ph_cdc_seq",+ ),("rejects_a_history_table_for_a_source_column_named_like_a_validity_column",{@@ -4587,6 +4599,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):[("turning_history_on_over_a_synced_column_is_refused",+ "valid_to","consolidated",None,{"cdc_table_mode": "both"},@@ -4594,6 +4607,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):),("turning_history_on_over_a_column_left_out_is_allowed",+ "valid_to","consolidated",["id"],{"cdc_table_mode": "both"},@@ -4601,17 +4615,43 @@ class TestSerializerLostUpdateProtection(APIBaseTest):),("adding_the_column_to_enabled_columns_is_refused",+ "valid_to","both",["id"],{"enabled_columns": ["id", "valid_to"]},True,),- ("editing_a_table_that_already_syncs_it_is_allowed", "both", None, {"enabled_columns": None}, False),+ (+ "editing_a_table_that_already_syncs_it_is_allowed",+ "valid_to",+ "both",+ None,+ {"enabled_columns": None},+ False,+ ),+ # Capture columns are refused on every CDC table, history or not.+ (+ "adding_a_capture_column_to_enabled_columns_is_refused",+ "_PH_DELETED",+ "consolidated",+ ["id"],+ {"enabled_columns": ["id", "_PH_DELETED"]},+ True,+ ),+ (+ "editing_a_table_that_already_syncs_a_capture_column_is_allowed",+ "_PH_DELETED",+ "consolidated",+ None,+ {"enabled_columns": None},+ False,+ ),])- def test_patch_checks_the_validity_columns_the_history_table_receives(+ def test_patch_checks_the_columns_a_cdc_sync_writes(self,_name: str,+ column: str,initial_mode: str,initial_enabled_columns: list[str] | None,request_body: dict,@@ -4622,7 +4662,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):)self.schema.sync_type_config["cdc_table_mode"] = initial_mode- self.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": "valid_to"}]}+ self.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": column}]}self.schema.enabled_columns = initial_enabled_columnsself.schema.save()@@ -4638,7 +4678,7 @@ class TestSerializerLostUpdateProtection(APIBaseTest):):serializer.is_valid(raise_exception=True)if refused:- with pytest.raises(ValidationError, match="valid_to"):+ with pytest.raises(ValidationError, match=column):serializer.save()else:serializer.save()diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex 883552c83881..56477cc0e485 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -5036,6 +5036,14 @@ class TestExternalDataSource(APIBaseTest):"consolidated",None,),+ (+ "capture_column_the_load_renames",+ [("id", "uuid", False), ("_PH_DELETED", "boolean", True)],+ {"tracking_link": ["id"]},+ "tracking_link (_ph_deleted)",+ "consolidated",+ None,+ ),("validity_column_with_a_history_table",[("id", "uuid", False), ("valid_to", "timestamp", True)],