:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): name new history tables _history with _ph_valid_* columns (#123551)
Daniel RCb1b8d259190c
diff --git a/products/data_warehouse/backend/presentation/views/table.py b/products/data_warehouse/backend/presentation/views/table.pyindex f8ec43401630..8b4d2ff99e8b 100644--- a/products/data_warehouse/backend/presentation/views/table.py+++ b/products/data_warehouse/backend/presentation/views/table.py@@ -692,11 +692,12 @@ class TableViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viewsets.Mold_csv_allow_double_quotes = instance.csv_allow_double_quoteswas_history_table = instance.is_history_table+ had_ph_history_naming = instance.has_ph_history_namingfor attr, value in validated_data.items():setattr(instance, attr, value)- # The sync owns this marker, and an `options` edit through this API must not drop it.+ # The sync owns these markers, and an `options` edit through this API must not drop them.if was_history_table:- instance.mark_as_history_table()+ instance.mark_as_history_table(ph_naming=had_ph_history_naming)if (instance._is_csv_format()diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.tsindex bd9eee6a6413..3984d4d69833 100644--- a/products/desktop/packages/api-client/src/generated.ts+++ b/products/desktop/packages/api-client/src/generated.ts@@ -11022,6 +11022,10 @@ export namespace Schemas {* Whether exact source-side column metadata is available for safe source-query projection.*/source_column_metadata_available: boolean;+ /**+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.+ */+ history_table_name: string | null;/*** Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.*/diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.pyindex 2ca5d6b15764..b1614ef14001 100644--- a/products/warehouse_sources/backend/facade/source_management.py+++ b/products/warehouse_sources/backend/facade/source_management.py@@ -31,6 +31,14 @@ _LAZY = {"purge_buffer_prefix": "cdc.buffer","is_history_table_mode": "cdc.batcher","reserved_column_collisions": "cdc.batcher",+ "history_naming_unless_taken": "cdc.history_table",+ "pin_history_naming": "cdc.history_table",+ "pinned_history_naming": "cdc.history_table",+ "new_history_naming": "cdc.history_table",+ "new_history_names_taken": "cdc.history_table",+ "history_table_name": "cdc.history_table",+ "history_table_ids": "cdc.history_table",+ "is_another_schemas_history_table": "cdc.history_table","reserved_column_names": "cdc.batcher","BUFFER_LANE": "cdc.snapshot_lane","CDC_RESET_PENDING_KEY": "cdc.snapshot_lane",diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex 73758015baab..bb690d085a01 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -195,6 +195,11 @@ HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset({"_ph_cdc_op", "_ph_cdc_seq",# Set in `options` when the sync registers a table as a schema's history table. Column names cannot# tell the two apart, because a source table can have its own `valid_from` and `valid_to` columns.HISTORY_TABLE_OPTION = "history_table"+# Set in `options` when the sync creates a history table named `<table>_history` with the validity+# columns `_ph_valid_from` / `_ph_valid_to`. A history table without it is older and keeps `_cdc`,+# `valid_from` and `valid_to`.+HISTORY_NAMING_OPTION = "history_naming"+PH_HISTORY_NAMING = "ph_validity"# Characters that carry quoting meaning once a column name is written into SQL. The structure@@ -617,8 +622,16 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletdef is_history_table(self) -> bool:return (self.options or {}).get(HISTORY_TABLE_OPTION) is True- def mark_as_history_table(self) -> None:- self.options = {**(self.options or {}), HISTORY_TABLE_OPTION: True}+ def mark_as_history_table(self, *, ph_naming: bool = False) -> None:+ self.options = {+ **(self.options or {}),+ HISTORY_TABLE_OPTION: True,+ **({HISTORY_NAMING_OPTION: PH_HISTORY_NAMING} if ph_naming else {}),+ }++ @property+ def has_ph_history_naming(self) -> bool:+ return (self.options or {}).get(HISTORY_NAMING_OPTION) == PH_HISTORY_NAMINGdef hidden_columns(self) -> frozenset[str]:"""The plumbing columns to keep out of the catalog for this table."""diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex 067527870a35..882f0bd4744b 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -74,7 +74,13 @@ from products.warehouse_sources.backend.facade.source_management import (filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,get_cdc_adapter,hand_reset_to_capture_if_sync_running,+ history_naming_unless_taken,+ history_table_name,+ is_another_schemas_history_table,is_history_table_mode,+ new_history_naming,+ pin_history_naming,+ pinned_history_naming,purge_buffer_prefix,repair_is_running,reserved_column_collisions,@@ -466,14 +472,30 @@ def _refuse_reserved_columns(new_mode: str | None,new_enabled_columns: list[str] | None,new_primary_keys: list[str] | None,+ resulting_config: dict[str, Any],) -> None:"""Capture and the history table write their own columns next to the source's, so a synced sourcecolumn that loads under one of those names fails the sync. Only a collision this request adds is- refused, so a table that already has one can still be edited or switched back."""+ refused, so a table that already has one can still be edited or switched back.++ Pins the naming the columns were checked against on `resulting_config`, so a later flag change cannot+ refuse a column this request accepted."""+ history = None+ resolved = None+ if is_history_table_mode(new_mode) or is_history_table_mode(previous_mode):+ history = resolved = history_naming_unless_taken(instance)+ kept_history = instance.sync_type == ExternalDataSchema.SyncType.CDC and is_history_table_mode(previous_mode)+ if history is None and is_history_table_mode(new_mode) and not kept_history:+ raise ValidationError(+ f"A history table for {instance.name} would share its name with another table of this "+ "source. Choose the consolidated table only for this table."+ )+ # A taken name leaves only the naming a new history table would get.+ history = history or new_history_naming(instance.source)columns = _source_column_names(instance)taken = reserved_column_collisions(columns,- reserved_column_names(history=is_history_table_mode(new_mode)),+ reserved_column_names(history if is_history_table_mode(new_mode) else None),enabled_columns=new_enabled_columns,primary_keys=new_primary_keys,incremental_field=instance.incremental_field,@@ -481,7 +503,7 @@ def _refuse_reserved_columns(if taken and instance.sync_type == ExternalDataSchema.SyncType.CDC:existing = reserved_column_collisions(columns,- reserved_column_names(history=is_history_table_mode(previous_mode)),+ reserved_column_names(history if is_history_table_mode(previous_mode) else None),enabled_columns=previous_enabled_columns,primary_keys=previous_primary_keys,incremental_field=instance.incremental_field,@@ -493,6 +515,8 @@ def _refuse_reserved_columns("PostHog writes. Rename the column on your database, leave it out of the synced columns, or change ""this table's sync method or output tables.")+ if resolved is not None and is_history_table_mode(new_mode) and pinned_history_naming(instance) is None:+ pin_history_naming(resulting_config, resolved)def schema_display_status(schema: ExternalDataSchema) -> str | None:@@ -686,6 +710,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersread_only=True,help_text="Whether exact source-side column metadata is available for safe source-query projection.",)+ history_table_name = serializers.SerializerMethodField(+ read_only=True,+ help_text="Name of the history table that keeps every version of each row, or `null` when the table "+ "keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.",+ )# `source` shadows DRF's reserved `Field.source` attribute, so mypy flags the assignment;# the runtime behaviour (a read-only SerializerMethodField backed by get_source) is correct.source = serializers.SerializerMethodField( # type: ignore[assignment]@@ -726,6 +755,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"row_filter_columns","available_columns","source_column_metadata_available",+ "history_table_name","source","api_version","api_version_deprecation",@@ -746,11 +776,16 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers"row_filter_columns","available_columns","source_column_metadata_available",+ "history_table_name","source","api_version_deprecation","user_access_level",]+ @extend_schema_field(serializers.CharField(allow_null=True))+ def get_history_table_name(self, schema: ExternalDataSchema) -> str | None:+ return history_table_name(schema)+@extend_schema_field({"type": "array",@@ -1374,6 +1409,20 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializersnew_mode=resulting_config.get("cdc_table_mode"),new_enabled_columns=validated_data.get("enabled_columns", instance.enabled_columns),new_primary_keys=resulting_config.get("primary_key_columns"),+ resulting_config=resulting_config,+ )+ instance.sync_type_config = resulting_config+ validated_data["sync_type_config"] = resulting_config++ # Its table and the history table would share one name and one storage folder.+ if (+ validated_data.get("should_sync")+ and not instance.should_sync+ and is_another_schemas_history_table(instance)+ ):+ raise ValidationError(+ f"{instance.name} has the same name as the history table of another table of this source, so it "+ "can't sync. Rename the table on your database to sync it.")should_sync = validated_data.get("should_sync", None)diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.pyindex d78078f82ddb..f569ef03f478 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.facade.source_management import (CDCSourceAdapter,PostgresSource,get_cdc_adapter,+ history_table_ids,repair_cdc_source,source_type_supports_cdc,)@@ -535,20 +536,16 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):except ExternalDataSchema.DoesNotExist:pass+ table_ids = history_table_ids(instance)+# Force CDC schemas to pick a new strategy by clearing sync_type and pausing.ExternalDataSchema.objects.filter(source=instance,sync_type=ExternalDataSchema.SyncType.CDC,).exclude(deleted=True).update(sync_type=None, should_sync=False)- # Soft-delete `_cdc` companion DataWarehouseTable rows so the next sync- # rebuilds them once the user picks a new strategy.- DataWarehouseTable.objects.filter(- external_data_source_id=instance.id,- team_id=self.team_id,- deleted=False,- name__endswith="_cdc",- ).update(deleted=True)+ # Soft-delete the history tables so the next sync rebuilds them once the user picks a new strategy.+ DataWarehouseTable.objects.filter(id__in=table_ids, team_id=self.team_id).update(deleted=True)# Clear ALL cdc_* keys from job_inputs — leaving stale engine identifiers# behind (e.g. `cdc_consistent_point`) would corrupt resume tracking ifdiff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.pyindex 7869fab3340b..426b96e8da73 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py@@ -73,7 +73,10 @@ from products.warehouse_sources.backend.facade.source_management import (get_cdc_adapter,is_history_table_mode,is_team_allowlisted_for_internal_hosts,+ new_history_names_taken,+ new_history_naming,new_source_requires_ssl,+ pin_history_naming,reserved_column_collisions,reserved_column_names,sql_schema_metadata,@@ -1390,6 +1393,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):# Pre-fetch PK column names for CDC tablespk_columns_by_table: dict[str, list[str]] = {}+ history_naming_config_by_schema: dict[str, dict[str, Any]] = {}if cdc_enabled:cdc_table_names_by_schema: dict[str, set[str]] = {}cdc_schema_name_by_location: dict[tuple[str, str], str] = {}@@ -1457,10 +1461,11 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):# Capture and the history table write their own columns next to the source's, so a synced# source column that loads under one of those names would fail the source's first sync.# Refuse before any replication state exists.+ new_history = new_history_naming(new_source_model)reserved_columns_by_table = {schema["name"]: reserved_column_collisions([column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],- reserved_column_names(history=is_history_table_mode(schema.get("cdc_table_mode"))),+ reserved_column_names(new_history if is_history_table_mode(schema.get("cdc_table_mode")) else None),enabled_columns=_payload_enabled_columns(schema),primary_keys=pk_columns_by_table.get(schema["name"]),)@@ -1485,6 +1490,37 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):},)+ synced_names = [+ schema["name"]+ for schema in payload_schemas+ if schema.get("should_sync", False) and isinstance(schema.get("name"), str)+ ]+ history_names = [+ schema["name"]+ for schema in payload_schemas+ if schema.get("sync_type") == "cdc"+ and schema.get("should_sync", False)+ and is_history_table_mode(schema.get("cdc_table_mode"))+ and isinstance(schema.get("name"), str)+ ]+ tables_with_taken_history_name = new_history_names_taken(+ new_source_model, new_history, history_names, synced_names+ )+ if tables_with_taken_history_name:+ new_source_model.delete()+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={+ "message": (+ "A history table would have the same name as another table you chose to sync. Choose "+ "the consolidated table only, or leave the other table out, for these tables: "+ f"{', '.join(tables_with_taken_history_name)}."+ )+ },+ )+ for name in history_names:+ pin_history_naming(history_naming_config_by_schema.setdefault(name, {}), new_history)+# Engine-side CDC resource setup runs after PK validation so we don't leave# replication state on the source for a config we're about to refuse.if cdc_enabled:@@ -1699,6 +1735,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):"schema_metadata": schema_metadata,"cdc_table_mode": cdc_table_mode,}+ sync_type_config.update(history_naming_config_by_schema.get(schema_name, {}))else:sync_type_config = {"schema_metadata": schema_metadata}diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.pyindex d79c5e7e0d08..7949de648778 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py@@ -46,10 +46,12 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters impor)from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_SEQ_COLUMN,- HISTORY_COLUMNS,+ HISTORY_NAMING,ChangeEventBatcher,+ HistoryNaming,enrich_delete_rows,enrich_toast_omitted_rows,+ history_columns,is_history_table_mode,reserved_column_collisions,)@@ -75,6 +77,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.errors importCDCSlotNotConfiguredError,classify_cdc_error,)+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import SourceHistoryNamingsfrom products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import has_engine_seqfrom products.warehouse_sources.backend.temporal.data_imports.cdc.naming import (CDC_EXTRACTION_WORKFLOW_ID_PREFIX,@@ -203,6 +206,7 @@ class CDCExtractActivity:self.split_key_columns: dict[str, list[str]] = {}# Missing entry = sync all columns; otherwise the set is the projection (always includes PKs).self.enabled_columns_by_table: dict[str, set[str]] = {}+ self.history_naming_by_table: dict[str, HistoryNaming] = {}self.adapter: typing.Any = Noneself.reader: typing.Any = Noneself.batcher: ChangeEventBatcher | None = None@@ -381,8 +385,9 @@ class CDCExtractActivity:# The batcher already refused the capture columns. The history table's own columns are# checked here, because only the schema says whether this table keeps history.- if is_history_table_mode(schema.cdc_table_mode):- taken = reserved_column_collisions(raw_table.column_names, HISTORY_COLUMNS)+ history = self.history_naming_by_table.get(table_name)+ if history is not None:+ taken = reserved_column_collisions(raw_table.column_names, history_columns(history))if taken:raise CDCReservedColumnError(f"Table {table_name} has history table column(s) {taken}")@@ -615,7 +620,17 @@ class CDCExtractActivity:})+ history_namings: SourceHistoryNamings | None = Nonefor schema in self.cdc_schemas:+ if is_history_table_mode(schema.cdc_table_mode):+ if history_namings is None:+ assert self.source is not None+ history_namings = SourceHistoryNamings(self.source)+ # A taken name fails this table's own import. Capture then refuses only the names every+ # history table reserves, so the source's other tables keep syncing.+ self.history_naming_by_table[schema.name] = (+ history_namings.naming_unless_taken(schema) or HISTORY_NAMING+ )enabled = schema.enabled_columns# `None` = sync all; `[]` = retain PKs + incremental only. Match the# invariant used by build_select_clause / pipeline_sync / filter_dwh_columns.diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.pyindex 2646fe7f0d96..ff0d01cc2fd6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py@@ -5,6 +5,8 @@ from collections.abc import Callable, Iterableimport pyarrow as pa+from posthog.dataclasses import frozen+from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnErrorfrom products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEventfrom products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention@@ -24,18 +26,45 @@ CDC_SEQ_COLUMN = "_ph_cdc_seq"# name.CDC_SEQ_PROVENANCE = {b"posthog_cdc": b"engine_position"}-# Suffix of the SCD2 companion table's resource name ({schema.name}_cdc).-CDC_COMPANION_SUFFIX = "_cdc"++@frozen+class HistoryNaming:+ """The table name suffix and the validity column names of one history table.++ A history table keeps the naming it was created with, so code reads it from the table and never+ from a constant.+ """++ suffix: str+ valid_from: str+ valid_to: str++ @property+ def validity_columns(self) -> frozenset[str]:+ return frozenset({self.valid_from, self.valid_to})+++LEGACY_HISTORY_NAMING = HistoryNaming(suffix="_cdc", valid_from="valid_from", valid_to="valid_to")+# The `_ph_` names leave a source's own `valid_from` and `valid_to` free to sync.+HISTORY_NAMING = HistoryNaming(suffix="_history", valid_from="_ph_valid_from", valid_to="_ph_valid_to")+++def history_naming_of(column_names: Iterable[str]) -> HistoryNaming:+ """The naming a stamped history batch or table was written with.++ `_ph_valid_from` is reserved on every history table, so only the stamp can put it there.+ """+ return HISTORY_NAMING if HISTORY_NAMING.valid_from in set(column_names) else LEGACY_HISTORY_NAMING-def companion_resource_name(schema_name: str) -> str:- """Storage name for a schema's `_cdc` companion table.+def companion_resource_name(schema_name: str, naming: HistoryNaming) -> str:+ """Storage name for a schema's history table.Keyed on the schema's `name`, never its resolved folder: the companion is CDC-only and staysself-consistent with its `name`-keyed snapshot seed. Capture, the snapshot seed and thebuffered consumer all write the same table, so they all resolve the name here."""- return f"{schema_name}{CDC_COMPANION_SUFFIX}"+ return f"{schema_name}{naming.suffix}"# Per-row list of source columns the change stream omitted because they are@@ -44,10 +73,6 @@ def companion_resource_name(schema_name: str) -> str:# writing to DeltaLake.TOAST_OMITTED_COLUMN = "_ph_toast_omitted"-# SCD Type 2 columns added to the _cdc history table-SCD2_VALID_FROM_COLUMN = "valid_from"-SCD2_VALID_TO_COLUMN = "valid_to"-# Values of `cdc_table_mode` that write the history table.CDC_HISTORY_TABLE_MODES: frozenset[str] = frozenset({"cdc_only", "both"})@@ -61,13 +86,20 @@ def is_history_table_mode(cdc_table_mode: object) -> bool:CAPTURE_COLUMNS: frozenset[str] = frozenset({CDC_OP_COLUMN, CDC_TIMESTAMP_COLUMN, CDC_SEQ_COLUMN, DELETED_COLUMN, DELETED_AT_COLUMN, TOAST_OMITTED_COLUMN})-# The columns that only the history table adds.-HISTORY_COLUMNS: frozenset[str] = frozenset({SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN})-def reserved_column_names(*, history: bool) -> frozenset[str]:- """Every name a CDC sync writes next to the source's own columns."""- return CAPTURE_COLUMNS | HISTORY_COLUMNS if history else CAPTURE_COLUMNS+def history_columns(naming: HistoryNaming) -> frozenset[str]:+ """The columns a history table adds to the capture columns.++ `_ph_valid_*` stays reserved on a legacy history table too, because `history_naming_of` reads the+ naming from it.+ """+ return HISTORY_NAMING.validity_columns | naming.validity_columns+++def reserved_column_names(history: HistoryNaming | None) -> frozenset[str]:+ """Every name a CDC sync writes next to the source's own columns, `history` when it keeps a history table."""+ return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history_columns(history)def reserved_column_collisions(@@ -89,20 +121,13 @@ def reserved_column_collisions(return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in reserved)-# Columns that are CDC/SCD2 metadata — always filled from the event itself,-# never overwritten when enriching DELETE rows with last-known source data.-_CDC_METADATA_COLUMNS: frozenset[str] = frozenset(- {- CDC_OP_COLUMN,- CDC_TIMESTAMP_COLUMN,- CDC_SEQ_COLUMN,- DELETED_COLUMN,- DELETED_AT_COLUMN,- TOAST_OMITTED_COLUMN,- SCD2_VALID_FROM_COLUMN,- SCD2_VALID_TO_COLUMN,- }-)+def metadata_columns(history: HistoryNaming | None) -> frozenset[str]:+ """Columns always filled from the change event itself, never from the last known row.++ `history` names the validity columns stamped on a history batch. A source column that only shares a+ validity name with another naming is data.+ """+ return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history.validity_columns# Micro-batch thresholds for CDC WAL processing. Flushing periodically@@ -230,6 +255,7 @@ def enrich_delete_rows(table: pa.Table,pk_columns: list[str],existing_rows: pa.Table | None = None,+ history: HistoryNaming | None = None,) -> pa.Table:"""Fill data columns on DELETE rows from the last known state.@@ -246,8 +272,10 @@ def enrich_delete_rows(The processor pre-filters `existing_rows` to only the PKs present in DELETEevents before calling this, so the materialization is bounded by the batch size.- Metadata columns (op, timestamp, deleted flags, valid_from/to) are always- kept from the DELETE event itself and are never overwritten.+ Metadata columns (op, timestamp, deleted flags, and the validity columns of+ `history` when the table is a stamped history batch) are always kept from the+ DELETE event itself and are never overwritten. A source column that only shares+ a validity name with another naming is data."""if not pk_columns or table.num_rows == 0:return table@@ -262,7 +290,8 @@ def enrich_delete_rows(return tablepk_set = set(present_pks)- table_data_cols = [col for col in table.column_names if col not in _CDC_METADATA_COLUMNS and col not in pk_set]+ metadata = metadata_columns(history)+ table_data_cols = [col for col in table.column_names if col not in metadata and col not in pk_set]# Determine extra columns that exist in existing_rows but not in the current table.# A standalone DELETE event may only carry PK columns — we add the missing data@@ -272,7 +301,7 @@ def enrich_delete_rows(extra_cols_from_existing = [colfor col in existing_rows.column_names- if col not in _CDC_METADATA_COLUMNS and col not in pk_set and col not in table.column_names+ if col not in metadata and col not in pk_set and col not in table.column_names]all_data_cols = table_data_cols + extra_cols_from_existing@@ -461,8 +490,8 @@ def enrich_toast_omitted_rows(return pa.table(new_columns, schema=pa.schema(new_fields))-def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:- """Add SCD Type 2 columns (valid_from, valid_to) to a raw CDC event table.+def build_scd2_table(pa_table: pa.Table, pk_columns: list[str], naming: HistoryNaming) -> pa.Table:+ """Add the SCD Type 2 columns of `naming` (valid_from, valid_to) to a raw CDC event table.valid_from = _ph_cdc_timestamp (the commit timestamp of this event)valid_to = _ph_cdc_timestamp of the next event for the same PK, or null@@ -475,7 +504,7 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:A two-step merge + append in the load processor closes previous "current"rows (sets valid_to) when a new batch is written for the same PK."""- taken = reserved_column_collisions(pa_table.column_names, HISTORY_COLUMNS)+ taken = reserved_column_collisions(pa_table.column_names, history_columns(naming))if taken:# Delta refuses a duplicate column name at write time; failing here names the column and# keeps a half-built batch out of the writer.@@ -489,9 +518,9 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:)if pa_table.num_rows == 0:- return pa_table.append_column(- pa.field(SCD2_VALID_FROM_COLUMN, ts_type), pa.array([], type=ts_type)- ).append_column(pa.field(SCD2_VALID_TO_COLUMN, ts_type), pa.array([], type=ts_type))+ return pa_table.append_column(pa.field(naming.valid_from, ts_type), pa.array([], type=ts_type)).append_column(+ pa.field(naming.valid_to, ts_type), pa.array([], type=ts_type)+ )ts_col = pa_table.column(CDC_TIMESTAMP_COLUMN).to_pylist()@@ -515,10 +544,10 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:valid_to[idx] = ts_col[next_idx]return pa_table.append_column(- pa.field(SCD2_VALID_FROM_COLUMN, ts_type),+ pa.field(naming.valid_from, ts_type),pa_table.column(CDC_TIMESTAMP_COLUMN),).append_column(- pa.field(SCD2_VALID_TO_COLUMN, ts_type),+ pa.field(naming.valid_to, ts_type),pa.array(valid_to, type=ts_type),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.pyindex 793fce39f5d3..45e140eb5a18 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py@@ -174,7 +174,7 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {),CDCErrorCategory.RESERVED_COLUMN_CONFLICT: ("A source table has a column that loads under a name PostHog writes for change data capture, "- "such as _ph_cdc_seq or _ph_deleted, or valid_from or valid_to on a table with a history table. "+ "such as _ph_cdc_seq or _ph_valid_from, or valid_from on an older history table that ends in _cdc. ""Rename the column on your database, leave it out of the synced columns, or choose another sync ""method for that table.",False,diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/history_table.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/history_table.pynew file mode 100644index 000000000000..b4124efc2845--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/history_table.py@@ -0,0 +1,276 @@+"""Find a schema's history table and the naming it was created with."""++from __future__ import annotations++from collections import defaultdict+from collections.abc import Iterable+from typing import Any+from uuid import UUID++from django.db.models import Q, QuerySet++from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.models.table import HISTORY_NAMING_OPTION, PH_HISTORY_NAMING, DataWarehouseTable+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,+ HistoryNaming,+ companion_resource_name,+ is_history_table_mode,+)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import (+ build_table_name,+ resolve_table_and_folder_names,+)+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_source_flag_enabled++# New history tables are named `<table>_history` with `_ph_valid_from` / `_ph_valid_to` while this flag is on+# for the source. Keep it off until every loader reads the validity names from the batch: an older loader+# appends such a batch without closing the open versions.+PH_HISTORY_NAMING_FLAG = "warehouse-ph-history-naming"+_LEGACY_HISTORY_NAMING = "legacy"+++class HistoryTableNameTakenError(Exception):+ """Another table of the source already has the catalog name a new history table would take.++ Two registrations under one name hide one of the tables from queries, and the companion registration+ would take the other table's record.+ """+++def not_another_schemas_table(tables: QuerySet[DataWarehouseTable], schema_id: UUID) -> QuerySet[DataWarehouseTable]:+ """Drop tables another schema syncs into.++ A source table can carry a history table's name, as `orders_cdc` next to `orders` does. A history+ table is linked to no schema, or to its own schema in `cdc_only` mode.+ """+ return tables.filter(Q(externaldataschema__isnull=True) | Q(externaldataschema__id=schema_id)).distinct()+++def new_history_naming(source: ExternalDataSource) -> HistoryNaming:+ """The naming a history table of this source created now gets."""+ return HISTORY_NAMING if is_source_flag_enabled(source, PH_HISTORY_NAMING_FLAG) else LEGACY_HISTORY_NAMING+++def pin_history_naming(sync_type_config: dict[str, Any], naming: HistoryNaming) -> None:+ """Record on the schema the naming its columns were checked against.++ The flag can change, or fail to evaluate, before the history table is registered. Capture would then+ refuse a source column the API accepted.+ """+ sync_type_config[HISTORY_NAMING_OPTION] = PH_HISTORY_NAMING if naming == HISTORY_NAMING else _LEGACY_HISTORY_NAMING+++def pinned_history_naming(schema: ExternalDataSchema) -> HistoryNaming | None:+ value = (schema.sync_type_config or {}).get(HISTORY_NAMING_OPTION)+ if value == PH_HISTORY_NAMING:+ return HISTORY_NAMING+ return LEGACY_HISTORY_NAMING if value == _LEGACY_HISTORY_NAMING else None+++def kept_history_naming(schema: ExternalDataSchema) -> HistoryNaming | None:+ """The naming of the history table the schema's mode writes, or None when it writes none."""+ return history_naming(schema) if is_history_table_mode(schema.cdc_table_mode) else None+++def registered_history_naming(schema: ExternalDataSchema) -> HistoryNaming | None:+ """The naming of the schema's registered history table, or None when it has none yet.++ A stamped `_history` table wins over a table under the legacy name, which a source table can also+ carry. A legacy table predates the stamp and the marker, so its name is all it has.+ """+ source = schema.source+ names = {+ build_table_name(source, companion_resource_name(schema.name, naming)): naming+ for naming in (HISTORY_NAMING, LEGACY_HISTORY_NAMING)+ }+ tables = not_another_schemas_table(+ DataWarehouseTable.objects.filter(+ team_id=schema.team_id, external_data_source_id=source.id, deleted=False, name__in=names+ ),+ schema.id,+ )+ return _naming_of_registered({names[table.name]: table for table in tables})+++def _naming_of_registered(tables_by_naming: dict[HistoryNaming, DataWarehouseTable]) -> HistoryNaming | None:+ current = tables_by_naming.get(HISTORY_NAMING)+ if current is not None and current.has_ph_history_naming:+ return HISTORY_NAMING+ return LEGACY_HISTORY_NAMING if LEGACY_HISTORY_NAMING in tables_by_naming else None+++class SourceHistoryNamings:+ """Resolves the history naming of a source's schemas from one read of its schemas and tables."""++ def __init__(self, source: ExternalDataSource) -> None:+ self._source = source+ schemas = list(ExternalDataSchema.objects.filter(team_id=source.team_id, source_id=source.id, deleted=False))+ self._schema_ids_by_table: defaultdict[UUID, set[UUID]] = defaultdict(set)+ # A sibling that neither syncs nor holds a table registers nothing under its name. Turning its sync on+ # later is refused by `is_another_schemas_history_table`.+ self._synced_schema_ids_by_name: defaultdict[str, set[UUID]] = defaultdict(set)+ for schema in schemas:+ if schema.table_id is not None:+ self._schema_ids_by_table[schema.table_id].add(schema.id)+ if schema.should_sync or schema.table_id is not None:+ names = resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name)+ self._synced_schema_ids_by_name[build_table_name(source, names.table_storage_name)].add(schema.id)+ self._tables_by_name: defaultdict[str, list[DataWarehouseTable]] = defaultdict(list)+ for table in DataWarehouseTable.objects.filter(+ team_id=source.team_id, external_data_source_id=source.id, deleted=False+ ):+ self._tables_by_name[table.name].append(table)+ self._new_naming: HistoryNaming | None = None++ def naming(self, schema: ExternalDataSchema) -> HistoryNaming:+ """The naming of the schema's history table: its registration, else its pin, else `new_history_naming`.++ Raises `HistoryTableNameTakenError` when a new history table would take the catalog name of another+ table of the source.+ """+ registered = self._registered(schema)+ if registered is not None:+ return registered+ naming = pinned_history_naming(schema) or self._naming_for_new_tables()+ new_name = build_table_name(self._source, companion_resource_name(schema.name, naming))+ if self._tables_by_name.get(new_name) or self._synced_schema_ids_by_name.get(new_name, set()) - {schema.id}:+ raise HistoryTableNameTakenError(f"{new_name} is already the name of another table of this source")+ return naming++ def naming_unless_taken(self, schema: ExternalDataSchema) -> HistoryNaming | None:+ try:+ return self.naming(schema)+ except HistoryTableNameTakenError:+ return None++ def _registered(self, schema: ExternalDataSchema) -> HistoryNaming | None:+ # A table another schema syncs into is that schema's own, as `orders_cdc` next to `orders` is.+ tables_by_naming = {+ naming: table+ for naming in (HISTORY_NAMING, LEGACY_HISTORY_NAMING)+ for table in self._tables_by_name.get(+ build_table_name(self._source, companion_resource_name(schema.name, naming)), []+ )+ if self._schema_ids_by_table.get(table.id, set()) <= {schema.id}+ }+ return _naming_of_registered(tables_by_naming)++ def _naming_for_new_tables(self) -> HistoryNaming:+ if self._new_naming is None:+ self._new_naming = new_history_naming(self._source)+ return self._new_naming+++def history_naming(schema: ExternalDataSchema) -> HistoryNaming:+ """`SourceHistoryNamings.naming` for one schema."""+ return SourceHistoryNamings(schema.source).naming(schema)+++def new_history_names_taken(+ source: ExternalDataSource,+ naming: HistoryNaming,+ history_schema_names: Iterable[str],+ synced_schema_names: Iterable[str],+) -> list[str]:+ """The schemas of a new source whose history table would take the catalog name of a synced schema."""+ taken = {build_table_name(source, name) for name in synced_schema_names}+ return sorted(+ name+ for name in history_schema_names+ if build_table_name(source, companion_resource_name(name, naming)) in taken+ )+++def history_naming_unless_taken(schema: ExternalDataSchema) -> HistoryNaming | None:+ """`history_naming`, or None when the new name belongs to another table of the source."""+ try:+ return history_naming(schema)+ except HistoryTableNameTakenError:+ return None+++def is_another_schemas_history_table(schema: ExternalDataSchema) -> bool:+ """Whether the schema's own table name belongs to the history table of another schema.++ A sibling that keeps history counts before its history table is registered, which happens only after+ its first snapshot loads.+ """+ source = schema.source+ name = build_table_name(+ source, resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name).table_storage_name+ )+ siblings = ExternalDataSchema.objects.filter(+ team_id=schema.team_id, source_id=schema.source_id, deleted=False, sync_type=ExternalDataSchema.SyncType.CDC+ ).exclude(id=schema.id)+ # Only a sibling whose history name under some naming matches pays for the registration lookup.+ candidates = [+ sibling+ for sibling in siblings+ if any(+ build_table_name(source, companion_resource_name(sibling.name, naming)) == name+ for naming in (HISTORY_NAMING, LEGACY_HISTORY_NAMING)+ )+ ]+ if any(history_table_name(sibling) == name for sibling in candidates):+ return True+ taken = (+ DataWarehouseTable.objects.filter(+ team_id=schema.team_id, external_data_source_id=schema.source_id, deleted=False, name=name+ )+ .exclude(externaldataschema__id=schema.id)+ .first()+ )+ if taken is None:+ return False+ return taken.is_history_table or name in _legacy_history_names(source, exclude_schema_id=schema.id)+++def _schema_names(source: ExternalDataSource, *, exclude_schema_id: UUID | None = None) -> list[str]:+ schemas = ExternalDataSchema.objects.filter(team_id=source.team_id, source_id=source.id, deleted=False)+ if exclude_schema_id is not None:+ schemas = schemas.exclude(id=exclude_schema_id)+ return list(schemas.values_list("name", flat=True))+++def _legacy_history_names(source: ExternalDataSource, *, exclude_schema_id: UUID | None = None) -> set[str]:+ return {+ build_table_name(source, companion_resource_name(name, LEGACY_HISTORY_NAMING))+ for name in _schema_names(source, exclude_schema_id=exclude_schema_id)+ }+++def history_tables(source: ExternalDataSource) -> list[DataWarehouseTable]:+ """The source's registered history tables.++ One that stopped syncing before the marker existed, such as one whose schema dropped its history+ table, carries only the legacy name of its schema. A table under a legacy name that a schema of that+ very name syncs into, as `orders_cdc` next to `orders` does, is that schema's own table.+ """+ candidates = DataWarehouseTable.objects.filter(+ team_id=source.team_id, external_data_source_id=source.id, deleted=False+ ).filter(Q(options__history_table=True) | Q(name__in=_legacy_history_names(source)))+ own_tables = {+ table_id: build_table_name(source, name)+ for table_id, name in ExternalDataSchema.objects.filter(+ team_id=source.team_id, source_id=source.id, deleted=False, table_id__isnull=False+ ).values_list("table_id", "name")+ }+ return [table for table in candidates if table.is_history_table or own_tables.get(table.id) != table.name]+++def history_table_ids(source: ExternalDataSource) -> list[UUID]:+ return [table.id for table in history_tables(source)]+++def history_table_name(schema: ExternalDataSchema) -> str | None:+ """The name of the history table the schema's mode writes, or None when it writes none.++ Skips the name check of `history_naming`, so a list of schemas costs one query per history table.+ """+ if not schema.keeps_history:+ return None+ naming = registered_history_naming(schema) or pinned_history_naming(schema) or new_history_naming(schema.source)+ return build_table_name(schema.source, companion_resource_name(schema.name, naming))diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.pyindex db4477c8a334..e70d07ef751b 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.py@@ -15,7 +15,7 @@ from __future__ import annotationsimport asynciofrom collections import Counter, defaultdict-from collections.abc import Callable+from collections.abc import Callable, Iterablefrom functools import partialfrom typing import TYPE_CHECKING, Any, castfrom urllib.parse import unquote@@ -29,8 +29,9 @@ from posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_SEQ_COLUMN,- SCD2_VALID_FROM_COLUMN,- SCD2_VALID_TO_COLUMN,+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,+ history_naming_of,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import normalize_column_name@@ -91,6 +92,7 @@ class LanePosition:# False above the position-row cap: `applied` then carries keys only, and a batch row is# matched on key and operation alone.content_matched: bool = True+ non_content_columns: frozenset[str] = LEGACY_HISTORY_NAMING.validity_columns# Reads the rows at the position into a resolved copy of this. Deferred so a tick whose# buffer holds nothing at the position — every idle tick — never reads them at all.load_applied: Callable[[], LanePosition] | None = None@@ -98,13 +100,21 @@ class LanePosition:EMPTY_POSITION = LanePosition(position=None, applied={}, key_columns=())-# Columns that never take part in a content comparison: the position and operation are the key,-# the SCD2 pair is stamped after replay, and the TOAST marker names columns to leave out per row.-NON_CONTENT_COLUMNS = frozenset({SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN})+def non_content_columns(table_columns: Iterable[str]) -> frozenset[str]:+ """Columns that never take part in a content comparison, beyond every `_ph_` column.-def content_columns(names: list[str]) -> list[str]:- return [n for n in names if not n.startswith("_ph_") and n not in NON_CONTENT_COLUMNS]+ The position and operation are the key, the SCD2 pair is stamped after replay, and the TOAST marker+ names columns to leave out per row. A legacy history table stamps `valid_from` / `valid_to`. A new+ one stamps `_ph_` names, so its `valid_from` and `valid_to` are the source's own and count.+ """+ if history_naming_of(table_columns) == HISTORY_NAMING:+ return frozenset()+ return LEGACY_HISTORY_NAMING.validity_columns+++def content_columns(names: list[str], non_content: frozenset[str]) -> list[str]:+ return [n for n in names if not n.startswith("_ph_") and n not in non_content]def _has_position_column(delta_table: deltalake.DeltaTable) -> bool:@@ -153,11 +163,13 @@ async def read_lane_position(present = {field.name for field in delta_table.schema().fields}columns = [name for name in key_columns if name in present]candidates = _files_at_position(add_actions, highest)+ non_content = non_content_columns(present)return LanePosition(position=highest,applied={},key_columns=tuple(columns),- load_applied=partial(_load_applied, delta_table, add_actions, highest, candidates, columns),+ non_content_columns=non_content,+ load_applied=partial(_load_applied, delta_table, add_actions, highest, candidates, columns, non_content),)@@ -167,6 +179,7 @@ def _load_applied(highest: int,candidates: dict[str, int],columns: list[str],+ non_content: frozenset[str],) -> LanePosition:"""The rows the table holds at `highest`, read from the candidate files a batch at a time.@@ -210,13 +223,15 @@ def _load_applied(applied={key: [placeholder] * count for key, count in counts.items()},key_columns=tuple(columns),content_matched=False,+ non_content_columns=non_content,)at_position = pa.Table.from_batches(batches, schema=schema)return LanePosition(position=highest,- applied=_group_by_identity(at_position, columns),+ applied=_group_by_identity(at_position, columns, non_content),key_columns=tuple(columns),- content_schema=at_position.select(content_columns(at_position.column_names)).schema,+ content_schema=at_position.select(content_columns(at_position.column_names, non_content)).schema,+ non_content_columns=non_content,)@@ -306,11 +321,13 @@ def _all_null(add_actions: pa.Table) -> set[str]:}-def _group_by_identity(rows: pa.Table, key_columns: list[str]) -> dict[tuple[Any, ...], list[dict[str, Any]]]:+def _group_by_identity(+ rows: pa.Table, key_columns: list[str], non_content: frozenset[str]+) -> dict[tuple[Any, ...], list[dict[str, Any]]]:if not key_columns or not rows.num_rows:return {}keys = _identities(rows, key_columns)- contents = rows.select(content_columns(rows.column_names)).to_pylist()+ contents = rows.select(content_columns(rows.column_names, non_content)).to_pylist()grouped: defaultdict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list)for key, content in zip(keys, contents):grouped[key].append(content)diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.pyindex 5a11203067b5..fa29f90ed3b6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py@@ -13,10 +13,11 @@ import pyarrow as pafrom posthog.dataclasses import frozenfrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (- _CDC_METADATA_COLUMNS,CDC_OP_COLUMN,CDC_SEQ_COLUMN,CDC_SEQ_PROVENANCE,+ HistoryNaming,+ metadata_columns,)# Verification is a diagnostic; cap it so a delete-heavy batch can't dominate the write path.@@ -43,9 +44,10 @@ class DeleteEnrichmentReport:return self.rows_with_nulled_columns == 0-def _data_columns(table: pa.Table, primary_keys: list[str]) -> list[str]:+def _data_columns(table: pa.Table, primary_keys: list[str], history: HistoryNaming | None) -> list[str]:pk_set = set(primary_keys)- return [c for c in table.column_names if c not in _CDC_METADATA_COLUMNS and c not in pk_set]+ metadata = metadata_columns(history)+ return [c for c in table.column_names if c not in metadata and c not in pk_set]def _pk_tuples(table: pa.Table, primary_keys: list[str]) -> list[tuple]:@@ -136,6 +138,7 @@ def verify_delete_enrichment(table: pa.Table,primary_keys: list[str],existing_rows: pa.Table | None,+ history: HistoryNaming | None = None,) -> DeleteEnrichmentReport:"""Report DELETE rows that would null a data column the target still holds.@@ -152,7 +155,7 @@ def verify_delete_enrichment(if not present_pks:return empty- data_cols = [c for c in _data_columns(table, present_pks) if c in existing_rows.column_names]+ data_cols = [c for c in _data_columns(table, present_pks, history) if c in existing_rows.column_names]if not data_cols:return emptydiff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.pyindex 91e13d302a94..13b2c1e36658 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.py@@ -34,8 +34,8 @@ from products.data_warehouse.backend.facade.api import aget_s3_clientfrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_OP_COLUMN,CDC_SEQ_COLUMN,- SCD2_VALID_TO_COLUMN,TOAST_OMITTED_COLUMN,+ HistoryNaming,build_scd2_table,companion_resource_name,)@@ -106,6 +106,7 @@ class CDCLane:resource_name: strwrite_mode: CDCWriteMode+ history: HistoryNaming | None = Nonedef serves_buffered_lane(schema: ExternalDataSchema) -> bool:@@ -150,23 +151,23 @@ def purge_buffer_before_handover(schema: ExternalDataSchema, logger: FilteringBopurge_buffer_prefix(schema.team_id, str(schema.id), logger, strict=True)-def served_lanes(schema: ExternalDataSchema) -> list[CDCLane]:- """The tables this schema's change stream feeds.+def served_lanes(schema: ExternalDataSchema, history: HistoryNaming | None) -> list[CDCLane]:+ """The tables this schema's change stream feeds. `history` names the history table, when the mode writes one.One entry per Delta table the mode writes, in the order a run writes them.An unrecognized mode returns nothing, which reads as "not a lane the buffer serves"."""- return [- CDCLane(- resource_name=(- companion_resource_name(schema.name)- if mode == COMPANION_WRITE_MODE- else consolidated_resource_name(schema)- ),- write_mode=mode,- )- for mode in _LANE_WRITE_MODES.get(schema.cdc_table_mode, ())- ]+ lanes: list[CDCLane] = []+ for mode in _LANE_WRITE_MODES.get(schema.cdc_table_mode, ()):+ if mode != COMPANION_WRITE_MODE:+ lanes.append(CDCLane(resource_name=consolidated_resource_name(schema), write_mode=mode))+ elif history is None:+ raise ValueError(f"CDC schema {schema.name} writes a history table, but no history naming was given")+ else:+ lanes.append(+ CDCLane(resource_name=companion_resource_name(schema.name, history), write_mode=mode, history=history)+ )+ return lanes# Slack when comparing an S3 mtime against a listing timestamp from our clock, so skew between the@@ -331,7 +332,9 @@ _PROOF_SEARCH_DEPTH = 10_PROOF_WINDOW = dt.timedelta(days=2)-def _history_transform(replay: ReplayFilter, key_columns: list[str]) -> Callable[[pa.Table], pa.Table]:+def _history_transform(+ replay: ReplayFilter, key_columns: list[str], history: HistoryNaming+) -> Callable[[pa.Table], pa.Table]:"""Replay first, then stamp the SCD2 validity columns onto what is left.Derived here rather than in the loader so the staged parquet is complete on its own: a loader@@ -349,16 +352,16 @@ def _history_transform(replay: ReplayFilter, key_columns: list[str]) -> Callabletable = replay.apply(table)if not table.num_rows:return table- # No guard on the columns already being there: a source that owns `valid_from` fails+ # No guard on the columns already being there: a source that owns a validity column fails# loudly inside `build_scd2_table`, where a skip would let the writer close rows against# customer data.- return build_scd2_table(table, key_columns)+ return build_scd2_table(table, key_columns, history)return _applyasync def build_output_lanes(- schema: ExternalDataSchema, job: ExternalDataJob, logger: FilteringBoundLogger+ schema: ExternalDataSchema, job: ExternalDataJob, logger: FilteringBoundLogger, history: HistoryNaming | None) -> tuple[list[OutputLane], int | None]:"""Every table this run writes, and the position below which the buffer is settled.@@ -374,7 +377,7 @@ async def build_output_lanes("""lanes: list[OutputLane] = []positions: list[int | None] = []- for index, lane in enumerate(served_lanes(schema)):+ for index, lane in enumerate(served_lanes(schema, history)):delta_table = await DeltaTableRef(lane.resource_name, job, logger, expect_missing=schema.table_id is None).get_delta_table()@@ -382,7 +385,7 @@ async def build_output_lanes(keys = [normalize_column_name(name) for name in schema.primary_key_columns or []]if delta_table is not None:# Before the read, so this run's own write is the one that carries the statistic.- await ensure_position_stats(delta_table, [*keys, *([SCD2_VALID_TO_COLUMN] if is_append else [])])+ await ensure_position_stats(delta_table, [*keys, *([lane.history.valid_to] if lane.history else [])])# Without primary keys an identity is just the operation, which matches rows the table# has never held. Ask for none instead: the lane then replays, which duplicates history# rather than losing it, and duplication is the direction that can be repaired.@@ -395,7 +398,7 @@ async def build_output_lanes(name=lane.resource_name,cdc_write_mode=lane.write_mode,billable=index == 0,- transform=_history_transform(replay, keys) if is_append else replay.apply,+ transform=_history_transform(replay, keys, lane.history) if lane.history else replay.apply,))floor = None if any(p is None for p in positions) else min(p for p in positions if p is not None)@@ -521,6 +524,7 @@ class ReplayFilter:# Taken from the position itself, so the batch is keyed exactly as the table was read.self._key_columns = list(position.key_columns)self._content_schema = position.content_schema+ self._non_content = position.non_content_columnsself._content_matched = position.content_matchedself._load_applied = position.load_appliedself.rows_skipped = 0@@ -541,6 +545,7 @@ class ReplayFilter:self._load_applied = Noneself._applied = {key: list(rows) for key, rows in resolved.applied.items()}self._content_schema = resolved.content_schema+ self._non_content = resolved.non_content_columnsself._content_matched = resolved.content_matcheddef _count_skipped(self, dropped: int, reason: str) -> None:@@ -612,7 +617,7 @@ class ReplayFilter:still compare different: dropping it would make a wrong match likelier, and a wrong matchis a lost change where a missed one is only a copy."""- names = content_columns(table.column_names)+ names = content_columns(table.column_names, self._non_content)if self._content_schema is None:return table.select(names).to_pylist(), set()columns: list[pa.ChunkedArray] = []diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.pyindex cecc7f5737a1..c15378ebdf82 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py@@ -89,6 +89,7 @@ class CDCSourceMixin(_BaseSource[ConfigType]):from products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobs import (retire_orphaned_companions,)+ from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import kept_history_namingfrom products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import hand_reset_to_capturefrom products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (CDCSourceManager,@@ -100,7 +101,8 @@ class CDCSourceMixin(_BaseSource[ConfigType]):served_lanes,)- if not served_lanes(schema):+ history = kept_history_naming(schema)+ if not served_lanes(schema, history):raise ValueError(f"CDC schema {schema.name} has cdc_table_mode {schema.cdc_table_mode!r}, which no buffer lane ""writes. Set it to 'consolidated', 'cdc_only' or 'both'."@@ -113,7 +115,7 @@ class CDCSourceMixin(_BaseSource[ConfigType]):# of that attempt failing later would leave a Completed job proving a listing nothing# drained.clear_listing(inputs.job_id, inputs.team_id)- first_lane = served_lanes(schema)[0]+ first_lane = served_lanes(schema, history)[0]return SourceResponse(name=first_lane.resource_name,items=lambda: iter(()),@@ -167,7 +169,7 @@ class CDCSourceMixin(_BaseSource[ConfigType]):# Every table this schema's changes feed, written from one read of the buffer. Each lane# carries where its own table stops, because a failed run can leave one ahead of the other.- lanes, deletion_floor = async_to_sync(build_output_lanes)(schema, job, inputs.logger)+ lanes, deletion_floor = async_to_sync(build_output_lanes)(schema, job, inputs.logger, history)manager = CDCSourceManager(inputs,inputs.logger,diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.pyindex 38deac34afb9..f711d760c24f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py@@ -37,13 +37,19 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importCDC_SEQ_COLUMN,CDC_TIMESTAMP_COLUMN,DELETED_COLUMN,+ HISTORY_NAMING,TOAST_OMITTED_COLUMN,companion_resource_name,enrich_delete_rows,enrich_toast_omitted_rows,+ history_naming_of,)from products.warehouse_sources.backend.temporal.data_imports.cdc.buffer import get_buffer_prefix-from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import LanePosition, content_columns+from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import (+ LanePosition,+ content_columns,+ non_content_columns,+)from products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import resolve_batchfrom products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (COMPANION_WRITE_MODE,@@ -336,11 +342,14 @@ def lane_position(rows: Sequence[Row], key_columns: Sequence[str] | None) -> Lanif position is None or not key_columns:return LanePosition(position=position, applied={})applied: defaultdict[Key, list[Row]] = defaultdict(list)+ non_content = non_content_columns({column for row in rows for column in row})for row in rows:if row.get(CDC_SEQ_COLUMN) == position:identity = tuple(row.get(column) for column in key_columns)- applied[identity].append({column: row[column] for column in content_columns(list(row))})- return LanePosition(position=position, applied=dict(applied), key_columns=tuple(key_columns))+ applied[identity].append({column: row[column] for column in content_columns(list(row), non_content)})+ return LanePosition(+ position=position, applied=dict(applied), key_columns=tuple(key_columns), non_content_columns=non_content+ )class Warehouse:@@ -374,7 +383,7 @@ class Warehouse:current = {self._table.key_of(row): row for row in self.history}existing = self._as_table(list(current.values()))batch = enrich_toast_omitted_rows(batch, self._keys, existing)- batch = enrich_delete_rows(batch, self._keys, existing)+ batch = enrich_delete_rows(batch, self._keys, existing, history_naming_of(batch.column_names))self.history.extend(_without_toast_marker(batch).to_pylist())def reseed(self, rows: dict[Key, Row]) -> None:@@ -531,6 +540,11 @@ class ConformanceKit:self._engine.installed(),patch(f"{_CDC}.activities.get_cdc_adapter", return_value=self._engine.adapter),patch(f"{_CDC}.activities.source_type_supports_cdc", return_value=True),+ # The kit registers no tables, so every history table is a new one.+ patch(+ f"{_CDC}.activities.SourceHistoryNamings",+ return_value=MagicMock(naming_unless_taken=MagicMock(return_value=HISTORY_NAMING)),+ ),patch(f"{_CDC}.activities.ExternalDataSource", source_model),patch.object(CDCExtractActivity, "_get_cdc_schemas", cdc_schemas),patch(f"{_CDC}.activities.update_sync_type_config_keys", merge_config),@@ -587,12 +601,12 @@ class ConformanceKit:schema, warehouse = self.schema(table), self._warehouses[table.name]self._lane_rows = {consolidated_resource_name(schema): lambda: list(warehouse.merged.values()),- companion_resource_name(schema.name): lambda: warehouse.history,+ companion_resource_name(schema.name, HISTORY_NAMING): lambda: warehouse.history,}inputs = SimpleNamespace(team_id=_TEAM_ID, schema_id=str(schema.id), job_id="job")async def drain() -> None:- lanes, floor = await build_output_lanes(schema, MagicMock(), self._log)+ lanes, floor = await build_output_lanes(schema, MagicMock(), self._log, HISTORY_NAMING)manager = CDCSourceManager(cast(SourceInputs, inputs), self._log, deletion_floor=floor, proof=self._proofs.get(table.name))diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.pyindex 1a11ed1f6cab..e01cd0aa7efc 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py@@ -15,10 +15,11 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importCDC_TIMESTAMP_COLUMN,DELETED_AT_COLUMN,DELETED_COLUMN,- SCD2_VALID_FROM_COLUMN,- SCD2_VALID_TO_COLUMN,+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,TOAST_OMITTED_COLUMN,ChangeEventBatcher,+ HistoryNaming,build_scd2_table,enrich_delete_rows,enrich_toast_omitted_rows,@@ -365,15 +366,15 @@ class TestBuildScd2Table:for i, (op, cols, ts) in enumerate(ops_cols_ts, start=1)]table = self._make_raw_table(events)- result = build_scd2_table(table, pk_columns)- assert result.column(SCD2_VALID_TO_COLUMN).to_pylist() == expected_valid_to- assert result.column(SCD2_VALID_FROM_COLUMN).to_pylist() == expected_valid_from+ result = build_scd2_table(table, pk_columns, HISTORY_NAMING)+ assert result.column(HISTORY_NAMING.valid_to).to_pylist() == expected_valid_to+ assert result.column(HISTORY_NAMING.valid_from).to_pylist() == expected_valid_fromdef test_valid_from_equals_cdc_timestamp(self):ts = datetime(2026, 3, 15, 10, 30, 0, tzinfo=UTC)table = self._make_raw_table([_make_event(timestamp=ts)])- result = build_scd2_table(table, ["id"])- assert result.column(SCD2_VALID_FROM_COLUMN)[0].as_py() == ts+ result = build_scd2_table(table, ["id"], HISTORY_NAMING)+ assert result.column(HISTORY_NAMING.valid_from)[0].as_py() == tsassert result.column(CDC_TIMESTAMP_COLUMN)[0].as_py() == ts@@ -444,6 +445,20 @@ class TestEnrichDeleteRows:assert result.column(DELETED_AT_COLUMN)[1].as_py() == ts_deleteassert result.column(DELETED_COLUMN)[1].as_py() is True+ def test_only_the_stamped_validity_columns_are_metadata(self):+ # A new history table frees `valid_from` for the source, so a delete must carry the source's+ # last value for it while keeping its own stamp.+ ts_insert = datetime(2025, 1, 1, tzinfo=UTC)+ ts_delete = datetime(2025, 6, 1, tzinfo=UTC)+ events = [+ _make_event(op="I", columns={"id": 1, "valid_from": "2020-01-01"}, timestamp=ts_insert),+ _make_event(op="D", columns={"id": 1}, timestamp=ts_delete),+ ]+ table = build_scd2_table(self._make_raw_table(events), ["id"], HISTORY_NAMING)+ result = enrich_delete_rows(table, ["id"], history=HISTORY_NAMING)+ assert result.column("valid_from")[1].as_py() == "2020-01-01"+ assert result.column(HISTORY_NAMING.valid_from)[1].as_py() == ts_delete+def test_empty_pk_columns_returns_unchanged(self):table = self._make_raw_table([_make_event(op="D", columns={"id": 1})])result = enrich_delete_rows(table, [])@@ -648,7 +663,7 @@ class TestSeqColumn:assert has_engine_seq(table)assert has_engine_seq(toasted)assert has_engine_seq(enriched)- assert has_engine_seq(build_scd2_table(enriched, ["id"]))+ assert has_engine_seq(build_scd2_table(enriched, ["id"], HISTORY_NAMING))def test_engine_seq_is_stamped_and_survives_a_parquet_round_trip(self):# The load side tells our column from a same-named source column by this stamp, and it@@ -705,29 +720,52 @@ class TestScd2TimestampType:})- result = build_scd2_table(table, ["id"])+ result = build_scd2_table(table, ["id"], HISTORY_NAMING)- assert result.schema.field(SCD2_VALID_FROM_COLUMN).type == ts- assert result.schema.field(SCD2_VALID_TO_COLUMN).type == ts- assert result.column(SCD2_VALID_TO_COLUMN).to_pylist()[0] is not None+ assert result.schema.field(HISTORY_NAMING.valid_from).type == ts+ assert result.schema.field(HISTORY_NAMING.valid_to).type == ts+ assert result.column(HISTORY_NAMING.valid_to).to_pylist()[0] is not Noneclass TestReservedColumnCollisions:@parameterized.expand([- ("exact_name", ["id", "valid_to"], True, None, None, ["valid_to"]),- ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], True, None, None, ["VALID_TO", "validFrom"]),- ("capture_name_the_load_renames", ["id", "_PH_CDC_SEQ"], False, None, None, ["_PH_CDC_SEQ"]),- ("validity_name_without_history", ["id", "valid_from"], False, None, None, []),- ("left_out_of_enabled_columns", ["id", "validFrom"], True, ["id"], None, []),- ("kept_as_the_incremental_field", ["id", "valid_from"], True, [], "valid_from", ["valid_from"]),+ ("legacy_validity_name", ["id", "valid_to"], LEGACY_HISTORY_NAMING, None, None, ["valid_to"]),+ (+ "legacy_name_the_load_renames",+ ["id", "validFrom", "VALID_TO"],+ LEGACY_HISTORY_NAMING,+ None,+ None,+ ["VALID_TO", "validFrom"],+ ),+ ("legacy_validity_name_on_a_new_table", ["id", "valid_from"], HISTORY_NAMING, None, None, []),+ (+ "new_validity_name_on_a_legacy_table",+ ["id", "_PH_VALID_FROM"],+ LEGACY_HISTORY_NAMING,+ None,+ None,+ ["_PH_VALID_FROM"],+ ),+ ("capture_name_the_load_renames", ["id", "_PH_CDC_SEQ"], None, None, None, ["_PH_CDC_SEQ"]),+ ("validity_name_without_history", ["id", "valid_from"], None, None, None, []),+ ("left_out_of_enabled_columns", ["id", "validFrom"], LEGACY_HISTORY_NAMING, ["id"], None, []),+ (+ "kept_as_the_incremental_field",+ ["id", "valid_from"],+ LEGACY_HISTORY_NAMING,+ [],+ "valid_from",+ ["valid_from"],+ ),])def test_names_the_source_columns_that_collide(self,_name: str,column_names: list[str],- history: bool,+ history: HistoryNaming | None,enabled_columns: list[str] | None,incremental_field: str | None,expected: list[str],@@ -735,7 +773,7 @@ class TestReservedColumnCollisions:assert (reserved_column_collisions(column_names,- reserved_column_names(history=history),+ reserved_column_names(history),enabled_columns=enabled_columns,primary_keys=["id"],incremental_field=incremental_field,@@ -760,8 +798,16 @@ class TestIsHistoryTableMode:class TestScd2ReservedColumns:- @parameterized.expand([(SCD2_VALID_FROM_COLUMN,), (SCD2_VALID_TO_COLUMN,)])- def test_a_source_column_named_like_the_stamp_fails_before_the_writer(self, taken):+ @parameterized.expand(+ [+ ("legacy_valid_from", LEGACY_HISTORY_NAMING, LEGACY_HISTORY_NAMING.valid_from),+ ("legacy_valid_to", LEGACY_HISTORY_NAMING, LEGACY_HISTORY_NAMING.valid_to),+ ("new_valid_from", HISTORY_NAMING, HISTORY_NAMING.valid_from),+ ]+ )+ def test_a_source_column_named_like_the_stamp_fails_before_the_writer(+ self, _name: str, history: HistoryNaming, taken: str+ ) -> None:# Delta refuses the duplicate name at write time, but by then a batch is half built and# the error names a qualified field, not the customer's column. A silent skip here is# worse still: the writer would close rows against the customer's own values.@@ -776,4 +822,4 @@ class TestScd2ReservedColumns:)with pytest.raises(CDCReservedColumnError, match=taken):- build_scd2_table(table, ["id"])+ build_scd2_table(table, ["id"], history)diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.pyindex 27b0d710a13e..aa6360c62ec7 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.py@@ -4,6 +4,7 @@ import pytestfrom unittest.mock import ANY, AsyncMock, MagicMock, call, patchfrom products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import LEGACY_HISTORY_NAMINGfrom products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import LanePositionfrom products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import ListingProoffrom products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs@@ -15,6 +16,7 @@ _MANAGER = "products.warehouse_sources.backend.temporal.data_imports.cdc.source__COMPANIONS = "products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobs"_SOURCE = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source"_SNAPSHOT_LANE = "products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane"+_HISTORY_TABLE = "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table"def _schema(**overrides) -> MagicMock:@@ -89,6 +91,11 @@ def _dispatch(patch(f"{_MANAGER}.completed_listing_proof", AsyncMock(return_value=proof)),patch(f"{_MANAGER}.buffer_expired_unread", expired or AsyncMock(return_value=False)),patch(f"{_SNAPSHOT_LANE}.hand_reset_to_capture", hand_reset or MagicMock()),+ # The schemas are mocks, so the registered naming cannot be read. Every history table here is legacy.+ patch(+ f"{_HISTORY_TABLE}.kept_history_naming",+ side_effect=lambda schema: LEGACY_HISTORY_NAMING if schema.cdc_table_mode in ("cdc_only", "both") else None,+ ),patch.object(PostgresSource, "make_ssh_tunnel_func", return_value=MagicMock()),):objects.select_related.return_value.get.return_value = schemadiff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.pyindex 10d20de3173c..ae0b7e550cd4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py@@ -34,7 +34,10 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importCDC_OP_COLUMN,CDC_SEQ_COLUMN,CDC_SEQ_PROVENANCE,+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,ChangeEventBatcher,+ HistoryNaming,)from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import (CDCErrorCategory,@@ -2342,22 +2345,28 @@ class TestBufferedIngressCapture:@parameterized.expand([- ("position_column", CDC_SEQ_COLUMN, "consolidated", True),- ("position_column_the_load_renames", "_PH_CDC_SEQ", "consolidated", True),- ("validity_column_with_a_history_table", "validFrom", "both", True),- ("validity_column_without_a_history_table", "valid_from", "consolidated", False),+ ("position_column", CDC_SEQ_COLUMN, None, True),+ ("position_column_the_load_renames", "_PH_CDC_SEQ", None, True),+ ("validity_column_on_a_legacy_history_table", "validFrom", LEGACY_HISTORY_NAMING, True),+ ("legacy_validity_column_on_a_new_history_table", "valid_from", HISTORY_NAMING, False),+ ("validity_column_without_a_history_table", "valid_from", None, False),])def test_a_source_column_named_like_a_written_column_fails_the_buffered_run(- self, _name: str, column: str, cdc_table_mode: str, refused: bool+ self, _name: str, column: str, history: HistoryNaming | None, refused: bool) -> None:# Written anyway, the file's name, ordering, and retry cleanup could derive from customer# data, and cleanup can then delete unconsumed files.source = _make_source()+ cdc_table_mode = "consolidated" if history is None else "both"schema = _make_schema("users", cdc_mode="streaming", cdc_table_mode=cdc_table_mode, source=source)events = [_make_event(op="I", position="0/100", columns={"id": 1, column: 42})]- with _capture_harness(source, [schema], events) as capture:+ with (+ patch(f"{_ACTIVITIES}.SourceHistoryNamings") as history_namings,+ _capture_harness(source, [schema], events) as capture,+ ):+ history_namings.return_value.naming_unless_taken.return_value = historyif refused:with pytest.raises(NonRetryableException, match="change data capture"):capture.extract()diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.pynew file mode 100644index 000000000000..c24b8ee21b43--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py@@ -0,0 +1,192 @@+import uuid++import pytest+from posthog.test.base import BaseTest+from unittest.mock import patch++from parameterized import parameterized++from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,+ HistoryNaming,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (+ HistoryTableNameTakenError,+ history_naming,+ history_tables,+ is_another_schemas_history_table,+ new_history_names_taken,+ pin_history_naming,+)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name++_PH_HISTORY_NAMING_FLAG = (+ "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"+)+++@patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)+class TestHistoryNaming(BaseTest):+ def setUp(self) -> None:+ super().setUp()+ self.source = ExternalDataSource.objects.create(+ team=self.team, source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), source_type="Postgres"+ )+ self.schema = ExternalDataSchema.objects.create(+ name="orders",+ team=self.team,+ source=self.source,+ sync_type=ExternalDataSchema.SyncType.CDC,+ sync_type_config={"cdc_table_mode": "both"},+ )++ def _table(self, storage_name: str, options: dict | None = None) -> DataWarehouseTable:+ return DataWarehouseTable.objects.create(+ team=self.team,+ name=build_table_name(self.source, storage_name),+ format="DeltaS3Wrapper",+ url_pattern=f"s3://bucket/{storage_name}",+ external_data_source=self.source,+ options=options or {},+ )++ @parameterized.expand(+ [+ ("no_history_table_yet", None, None, HISTORY_NAMING),+ ("legacy_table_without_the_marker", "orders_cdc", {}, LEGACY_HISTORY_NAMING),+ ("legacy_table_with_the_marker", "orders_cdc", {"history_table": True}, LEGACY_HISTORY_NAMING),+ (+ "stamped_new_table",+ "orders_history",+ {"history_table": True, "history_naming": "ph_validity"},+ HISTORY_NAMING,+ ),+ ]+ )+ def test_reads_the_naming_from_the_registered_table(+ self, _name: str, storage_name: str | None, options: dict | None, expected: object+ ) -> None:+ if storage_name is not None:+ self._table(storage_name, options)++ assert history_naming(self.schema) == expected++ def test_a_new_history_table_keeps_the_legacy_naming_until_the_flag_is_on(self) -> None:+ with patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: False):+ assert history_naming(self.schema) == LEGACY_HISTORY_NAMING++ @parameterized.expand(+ [("pinned_new_flag_off", HISTORY_NAMING, False), ("pinned_legacy_flag_on", LEGACY_HISTORY_NAMING, True)]+ )+ def test_a_pinned_naming_wins_over_the_flag(self, _name: str, pinned: HistoryNaming, flag_on: bool) -> None:+ pin_history_naming(self.schema.sync_type_config, pinned)+ self.schema.save()++ with patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: flag_on):+ assert history_naming(self.schema) == pinned++ def test_a_stamped_new_table_wins_over_a_legacy_named_table(self) -> None:+ # An unlinked `orders_cdc` can be a source table that has not synced yet. Taking the legacy naming+ # for it would stop writing the stamped history table.+ self._table("orders_history", {"history_table": True, "history_naming": "ph_validity"})+ self._table("orders_cdc")++ assert history_naming(self.schema) == HISTORY_NAMING++ def test_a_sibling_whose_catalog_name_differs_does_not_block_history(self) -> None:+ # `ordersHistory` normalizes to the same folder leaf, but each schema's folder carries its id.+ ExternalDataSchema.objects.create(name="ordersHistory", team=self.team, source=self.source, should_sync=True)++ assert history_naming(self.schema) == HISTORY_NAMING++ @parameterized.expand(+ [("new_naming", HISTORY_NAMING, "orders_history"), ("legacy_naming", LEGACY_HISTORY_NAMING, "orders_cdc")]+ )+ def test_new_history_names_taken_compares_the_payload_of_a_new_source(+ self, _name: str, naming: HistoryNaming, sibling: str+ ) -> None:+ taken = new_history_names_taken(self.source, naming, ["orders", "items"], ["orders", sibling, "items"])++ assert taken == ["orders"]++ def test_a_sibling_source_table_named_like_the_legacy_table_is_not_its_history(self) -> None:+ sibling = ExternalDataSchema.objects.create(name="orders_cdc", team=self.team, source=self.source)+ sibling.table = self._table("orders_cdc")+ sibling.save()++ assert history_naming(self.schema) == HISTORY_NAMING++ def test_refuses_a_name_another_table_holds(self) -> None:+ self._table("orders_history")++ with pytest.raises(HistoryTableNameTakenError):+ history_naming(self.schema)++ @parameterized.expand([("syncing", True, True), ("discovered_only", False, False)])+ def test_refuses_a_name_a_sibling_table_syncs_under(self, _name: str, should_sync: bool, refused: bool) -> None:+ ExternalDataSchema.objects.create(+ name="orders_history", team=self.team, source=self.source, should_sync=should_sync+ )++ if refused:+ with pytest.raises(HistoryTableNameTakenError):+ history_naming(self.schema)+ else:+ assert history_naming(self.schema) == HISTORY_NAMING++ @parameterized.expand(+ [+ ("new_table", "orders_history", {"history_table": True, "history_naming": "ph_validity"}),+ # A history table that stopped syncing before the marker existed never got it.+ ("unmarked_legacy_table", "orders_cdc", {}),+ ]+ )+ def test_a_schema_cannot_take_the_name_of_another_schemas_history_table(+ self, _name: str, storage_name: str, options: dict+ ) -> None:+ self._table(storage_name, options)+ sibling = ExternalDataSchema.objects.create(name=storage_name, team=self.team, source=self.source)++ assert is_another_schemas_history_table(sibling)+ assert not is_another_schemas_history_table(self.schema)++ # A schema renamed by the multi-schema migration keeps its pinned catalog name, not the name of its row.+ @parameterized.expand([("unpinned", None, True), ("pinned_to_old_name", "orders_history", False)])+ def test_the_reverse_guard_compares_the_catalog_name(+ self, _name: str, s3_folder_name: str | None, refused: bool+ ) -> None:+ # `orders` would own `orders_history` itself.+ self.schema.sync_type_config = {"cdc_table_mode": "consolidated"}+ self.schema.save()+ ExternalDataSchema.objects.create(+ name="public.orders",+ team=self.team,+ source=self.source,+ sync_type=ExternalDataSchema.SyncType.CDC,+ sync_type_config={"cdc_table_mode": "both"},+ )+ sibling = ExternalDataSchema.objects.create(+ name="public.orders_history", team=self.team, source=self.source, s3_folder_name=s3_folder_name+ )++ assert is_another_schemas_history_table(sibling) == refused++ def test_history_tables_finds_marked_and_unmarked_legacy_tables(self) -> None:+ legacy = self._table("orders_cdc")+ marked = self._table("other_history", {"history_table": True})+ self._table("orders")+ self._table("customer_cdc")+ # A discovered `orders_cdc` source table that does not sync owns no table, so the legacy table above+ # is still the history of `orders`.+ ExternalDataSchema.objects.create(name="orders_cdc", team=self.team, source=self.source)+ # `items_cdc` syncs into its own table, which is not the history of `items`.+ ExternalDataSchema.objects.create(name="items", team=self.team, source=self.source)+ ExternalDataSchema.objects.create(+ name="items_cdc", team=self.team, source=self.source, table=self._table("items_cdc")+ )++ assert set(history_tables(self.source)) == {legacy, marked}diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.pyindex 3bc218144156..50d6e1976a2a 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.py@@ -6,7 +6,12 @@ import deltalakeimport pyarrow.dataset as pa_dsfrom products.warehouse_sources.backend.temporal.data_imports.cdc import lane_position-from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import CDC_OP_COLUMN, CDC_SEQ_COLUMN+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (+ CDC_OP_COLUMN,+ CDC_SEQ_COLUMN,+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,+)from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import (STATS_COLUMNS_PROPERTY,ensure_position_stats,@@ -95,6 +100,27 @@ class TestReadLanePosition:assert position.content_schema is not Noneassert position.content_schema.names == ["id", *(f"c{i}" for i in range(_WIDE))]+ @pytest.mark.parametrize(+ ("history", "compared"),+ [(HISTORY_NAMING, ["id", "valid_from"]), (LEGACY_HISTORY_NAMING, ["id"])],+ ids=["new_table", "legacy_table"],+ )+ async def test_only_the_stamped_validity_columns_stay_out_of_the_comparison(self, tmp_path, history, compared):+ # A new table frees `valid_from` for the source. Left out, two changes to one key that differ+ # only there would compare equal, and the second would be dropped as already written.+ rows = _rows([30], wide=False, ids=[1])+ if history == HISTORY_NAMING:+ rows = rows.append_column("valid_from", pa.array(["2020-01-01"]))+ stamp = pa.array([None], pa.timestamp("us"))+ rows = rows.append_column(history.valid_from, stamp).append_column(history.valid_to, stamp)+ path = str(tmp_path / "t")+ deltalake.write_deltalake(path, rows, configuration={STATS_COLUMNS_PROPERTY: CDC_SEQ_COLUMN})++ position = await _resolved(deltalake.DeltaTable(path))++ assert position.content_schema is not None+ assert position.content_schema.names == compared+async def test_rows_at_the_position_are_gathered_across_the_files_holding_them(self, tmp_path):path = tmp_path / "t"_write(path, [10, 30], ids=[1, 2])diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.pyindex f308b9fb2943..39dd4b249f88 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.py@@ -14,8 +14,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importCDC_SEQ_COLUMN,CDC_SEQ_PROVENANCE,CDC_TIMESTAMP_COLUMN,- SCD2_VALID_FROM_COLUMN,- SCD2_VALID_TO_COLUMN,+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,TOAST_OMITTED_COLUMN,)from products.warehouse_sources.backend.temporal.data_imports.cdc.buffer import build_buffer_file_name@@ -306,30 +306,41 @@ class TestServedLanes:@parameterized.expand([("consolidated", "consolidated", [CDCLane(resource_name="users", write_mode="incremental_merge")]),- ("cdc_only", "cdc_only", [CDCLane(resource_name="users_cdc", write_mode="scd2_append")]),+ (+ "cdc_only",+ "cdc_only",+ [CDCLane(resource_name="users_history", write_mode="scd2_append", history=HISTORY_NAMING)],+ ),("both","both",[CDCLane(resource_name="users", write_mode="incremental_merge"),- CDCLane(resource_name="users_cdc", write_mode="scd2_append"),+ CDCLane(resource_name="users_history", write_mode="scd2_append", history=HISTORY_NAMING),],),])def test_a_table_mode_names_the_tables_its_changes_feed(self, _name, table_mode, expected):- assert served_lanes(_schema(cdc_table_mode=table_mode)) == expected+ assert served_lanes(_schema(cdc_table_mode=table_mode), HISTORY_NAMING) == expecteddef test_an_unrecognized_table_mode_feeds_nothing(self):# Fail closed: a mode this module cannot write must not reach the buffer at all.- assert served_lanes(_schema(cdc_table_mode="something_new")) == []+ assert served_lanes(_schema(cdc_table_mode="something_new"), HISTORY_NAMING) == []- def test_the_companion_is_named_off_the_schema_not_the_folder(self):+ @parameterized.expand(+ [+ ("new_table", HISTORY_NAMING, "public.users_history"),+ ("legacy_table", LEGACY_HISTORY_NAMING, "public.users_cdc"),+ ]+ )+ def test_the_companion_is_named_off_the_schema_not_the_folder(self, _name, history, expected):# The consolidated lane follows the snapshot's resolved folder, which diverges from `name`# for a row renamed bare to qualified. Following it here would append history into a table- # no query reads — the companion is keyed on `name`, like its snapshot seed.+ # no query reads — the companion is keyed on `name`, like its snapshot seed. A legacy history+ # table keeps writing under its own name.schema = _schema(name="public.users", resolved_s3_folder_name="users", cdc_table_mode="both")- assert [lane.resource_name for lane in served_lanes(schema)] == ["users", "public.users_cdc"]+ assert [lane.resource_name for lane in served_lanes(schema, history)] == ["users", expected]class TestSnapshotCapture:@@ -816,15 +827,15 @@ class TestBuildOutputLanes:AsyncMock(side_effect=positions),),):- return await build_output_lanes(schema, MagicMock(), AsyncMock())+ return await build_output_lanes(schema, MagicMock(), AsyncMock(), HISTORY_NAMING)async def test_both_writes_two_tables_in_one_run(self):lanes, _ = await self._build(_schema(cdc_table_mode="both"), [_NO_POSITION, _NO_POSITION])- assert [lane.name for lane in lanes] == ["users", "users_cdc"]+ assert [lane.name for lane in lanes] == ["users", "users_history"]assert [lane.cdc_write_mode for lane in lanes] == [CONSOLIDATED_WRITE_MODE, COMPANION_WRITE_MODE]- @parameterized.expand([("consolidated", "consolidated", "users"), ("cdc_only", "cdc_only", "users_cdc")])+ @parameterized.expand([("consolidated", "consolidated", "users"), ("cdc_only", "cdc_only", "users_history")])async def test_a_single_table_mode_bills_its_only_lane(self, _name, table_mode, expected):lanes, _ = await self._build(_schema(cdc_table_mode=table_mode), [_NO_POSITION])@@ -833,7 +844,7 @@ class TestBuildOutputLanes:async def test_both_bills_the_consolidated_lane_only(self):lanes, _ = await self._build(_schema(cdc_table_mode="both"), [_NO_POSITION, _NO_POSITION])- assert [(lane.name, lane.billable) for lane in lanes] == [("users", True), ("users_cdc", False)]+ assert [(lane.name, lane.billable) for lane in lanes] == [("users", True), ("users_history", False)]async def test_only_the_append_lane_is_asked_for_identity(self):# Asserted on the call, not on hand-fed positions: if both lanes asked for identity the@@ -855,14 +866,14 @@ class TestBuildOutputLanes:"products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager.read_lane_position", reads),):- await build_output_lanes(_schema(cdc_table_mode="both"), MagicMock(), AsyncMock())+ await build_output_lanes(_schema(cdc_table_mode="both"), MagicMock(), AsyncMock(), HISTORY_NAMING)merge_keys, append_keys = (call.kwargs["key_columns"] for call in reads.call_args_list)assert merge_keys is Noneassert append_keys == ["id", CDC_OP_COLUMN]# The history table's own merge predicates on valid_to, so it keeps its pruning too.- assert SCD2_VALID_TO_COLUMN in stats.call_args_list[1].args[1]- assert SCD2_VALID_TO_COLUMN not in stats.call_args_list[0].args[1]+ assert HISTORY_NAMING.valid_to in stats.call_args_list[1].args[1]+ assert HISTORY_NAMING.valid_to not in stats.call_args_list[0].args[1]async def test_a_lane_with_no_primary_keys_asks_for_no_identity(self):# Otherwise the identity is the operation alone, which matches rows the table never held.@@ -882,7 +893,7 @@ class TestBuildOutputLanes:),):await build_output_lanes(- _schema(cdc_table_mode="cdc_only", primary_key_columns=[]), MagicMock(), AsyncMock()+ _schema(cdc_table_mode="cdc_only", primary_key_columns=[]), MagicMock(), AsyncMock(), HISTORY_NAMING)assert reads.call_args.kwargs["key_columns"] is None@@ -907,9 +918,9 @@ class TestBuildOutputLanes:assert history.column("id").to_pylist() == [4]# Stamped by the lane, not the loader: a loader on the previous release has no SCD2 step# and would append these rows with no validity at all.- assert SCD2_VALID_FROM_COLUMN in history.column_names- assert SCD2_VALID_TO_COLUMN in history.column_names- assert SCD2_VALID_FROM_COLUMN not in merged.column_names+ assert HISTORY_NAMING.valid_from in history.column_names+ assert HISTORY_NAMING.valid_to in history.column_names+ assert HISTORY_NAMING.valid_from not in merged.column_namesasync def test_the_floor_is_the_lowest_position_any_table_holds(self):_, floor = await self._build(diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.pyindex e58e64c17435..f7d96399faa2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py@@ -183,10 +183,9 @@ async def _seed_cdc_companion_from_snapshot(CDC_TIMESTAMP_COLUMN,DELETED_AT_COLUMN,DELETED_COLUMN,- SCD2_VALID_FROM_COLUMN,- SCD2_VALID_TO_COLUMN,companion_resource_name as build_companion_resource_name,)+ from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import history_namingfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableReffrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import DeltaWriterfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchema@@ -194,6 +193,7 @@ async def _seed_cdc_companion_from_snapshot(snapshot_dt = await snapshot_delta_table_ref.get_delta_table()if snapshot_dt is None:return+ history = await database_sync_to_async_pool(history_naming)(schema)dataset = await asyncio.to_thread(snapshot_dt.to_pyarrow_dataset)@@ -203,12 +203,11 @@ async def _seed_cdc_companion_from_snapshot(CDC_TIMESTAMP_COLUMN,DELETED_COLUMN,DELETED_AT_COLUMN,- SCD2_VALID_FROM_COLUMN,- SCD2_VALID_TO_COLUMN,+ *history.validity_columns,}read_columns = [c for c in dataset.schema.names if c not in cdc_meta_cols]- companion_resource_name = build_companion_resource_name(schema.name)+ companion_resource_name = build_companion_resource_name(schema.name, history)companion_ref = DeltaTableRef(resource_name=companion_resource_name,job=job,@@ -254,8 +253,8 @@ async def _seed_cdc_companion_from_snapshot(.append_column(pa.field(CDC_TIMESTAMP_COLUMN, ts_type), pa.array([epoch_us] * n, type=ts_type)).append_column(pa.field(DELETED_COLUMN, pa.bool_()), pa.array([False] * n, type=pa.bool_())).append_column(pa.field(DELETED_AT_COLUMN, ts_type), pa.array([None] * n, type=ts_type))- .append_column(pa.field(SCD2_VALID_FROM_COLUMN, ts_type), pa.array([epoch_us] * n, type=ts_type))- .append_column(pa.field(SCD2_VALID_TO_COLUMN, ts_type), pa.array([None] * n, type=ts_type))+ .append_column(pa.field(history.valid_from, ts_type), pa.array([epoch_us] * n, type=ts_type))+ .append_column(pa.field(history.valid_to, ts_type), pa.array([None] * n, type=ts_type)))# Plain append — the companion table is freshly reset so there are no existingdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.pyindex edb2a2906a3e..b46552457221 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py@@ -34,9 +34,18 @@ from products.warehouse_sources.backend.models.external_data_schema import (update_sync_type_config_keys,)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource-from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable+from products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.models.util import hogql_type_name_for_clickhouse_type-from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import companion_resource_name+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,+ companion_resource_name,+ history_naming_of,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (+ HistoryTableNameTakenError,+ not_another_schemas_table,+)from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConventionfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry import (retry_on_operational_error,@@ -292,20 +301,24 @@ def _record_storage_delta(job: ExternalDataJob, storage_delta_mib: Optional[floadef own_linked_table(schema: ExternalDataSchema, pipeline: ExternalDataSource) -> DataWarehouseTable | None:- """The schema's linked table, unless the link is its `_cdc` companion.+ """The schema's linked table, unless the link is its history table.- cdc_only links the schema to its companion table. Reusing that link after a switch to a mode that- writes the consolidated table would publish the consolidated data under the companion's record,+ cdc_only links the schema to its history table. Reusing that link after a switch to a mode that+ writes the consolidated table would publish the consolidated data under the history table's record,and the consolidated table would never get a record of its own. A pinned folder can give the- schema's own table the companion's name, and then the link is right.+ schema's own table the legacy history name, and then the link is right."""table = schema.tableif table is None:return Nonenames = resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name)- table_name = build_table_name(pipeline, names.table_storage_name)- companion_name = build_table_name(pipeline, companion_resource_name(schema.name))- return None if table.name == companion_name != table_name else table+ if table.name == build_table_name(pipeline, names.table_storage_name):+ return table+ if table.is_history_table:+ return None+ # A history table registered before the marker existed carries only its legacy name.+ legacy_name = build_table_name(pipeline, companion_resource_name(schema.name, LEGACY_HISTORY_NAMING))+ return None if table.name == legacy_name else tabledef find_orphan_table(schema: ExternalDataSchema, source: ExternalDataSource) -> DataWarehouseTable | None:@@ -685,12 +698,19 @@ async def register_cdc_companion_table(# when the cluster is degraded, so holding a Postgres transaction open across them is# what produced minutes-long "idle in transaction" connections.# Find existing companion table (not schema.table) by name- companion_table: DataWarehouseTable | None = DataWarehouseTable.objects.filter(+ same_name = DataWarehouseTable.objects.filter(team_id=team_id,name=companion_table_name,external_data_source_id=job.pipeline.id,deleted=False,- ).first()+ )+ companion_table: DataWarehouseTable | None = not_another_schemas_table(same_name, schema_id).first()+ if companion_table is None and same_name.exists():+ # Another schema's table has this name. Taking its record would point it at this history,+ # and a second record of the same name would hide one of them.+ raise HistoryTableNameTakenError(+ f"{companion_table_name} is already the name of another table of this source"+ )previous_queryable_folder = companion_table.queryable_folder if companion_table else Nonestorage_delta_mib = _storage_delta_mib(companion_table, live_size_mib)@@ -717,13 +737,17 @@ async def register_cdc_companion_table()else:logger.debug(f"Creating CDC companion table: {companion_table_name}")- companion_table = DataWarehouseTable.objects.create(+ companion_table = DataWarehouseTable(external_data_source_id=job.pipeline.id,created_via=DataWarehouseTableCreatedVia.SOURCE,size_in_s3_mib=live_size_mib,- options={HISTORY_TABLE_OPTION: True},**table_params,)+ # The batches carry the validity columns of the naming this run chose for the table.+ companion_table.mark_as_history_table(+ ph_naming=history_naming_of(table_schema_dict or {}) == HISTORY_NAMING+ )+ companion_table.save()_record_storage_delta(job, storage_delta_mib)_record_query_folder_pointer(schema_id, team_id, previous_queryable_folder, queryable_folder, run_id, loggerdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.pyindex b7112f04080e..a29ec944b343 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py@@ -29,11 +29,10 @@ from products.warehouse_sources.backend.models.external_data_schema import Exterfrom products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_OP_COLUMN,- SCD2_VALID_FROM_COLUMN,- SCD2_VALID_TO_COLUMN,TOAST_OMITTED_COLUMN,enrich_delete_rows,enrich_toast_omitted_rows,+ history_naming_of,)from products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import (has_engine_seq,@@ -260,21 +259,22 @@ def _enrich_cdc_rows(# For SCD2 tables, keep only "current" rows (valid_to IS NULL) so we# enrich with the most recent state rather than a historical one.+ history = history_naming_of(pa_table.column_names) if cdc_write_mode == "scd2_append" else Noneif (- cdc_write_mode == "scd2_append"+ history is not Noneand existing_rows.num_rows > 0- and SCD2_VALID_TO_COLUMN in existing_rows.column_names+ and history.valid_to in existing_rows.column_names):- existing_rows = existing_rows.filter(pc.is_null(existing_rows.column(SCD2_VALID_TO_COLUMN)))+ existing_rows = existing_rows.filter(pc.is_null(existing_rows.column(history.valid_to)))# TOAST fill first so DELETE enrichment copies resolved values,# not the nulls standing in for omitted columns.pa_table = enrich_toast_omitted_rows(pa_table, primary_keys, existing_rows)- pa_table = enrich_delete_rows(pa_table, primary_keys, existing_rows)+ pa_table = enrich_delete_rows(pa_table, primary_keys, existing_rows, history)# Only place the previous state is still in hand to compare against.if verify_deletes and delete_key_set:- report = verify_delete_enrichment(pa_table, present_pks, existing_rows)+ report = verify_delete_enrichment(pa_table, present_pks, existing_rows, history)if not report.ok:CDC_DELETE_ENRICHMENT_VIOLATIONS_TOTAL.inc(report.rows_with_nulled_columns)logger.warning(@@ -1661,10 +1661,10 @@ def _process_message_reported(timer.step("write"),DELTA_WRITE_DURATION_SECONDS.labels(write_type="scd2_append").time(),):+ # The lane stamped the batch with its table's validity columns, so the batch names them.+ history = history_naming_of(pa_table.column_names)scd2_writer = Scd2DeltaWriter(- delta_table_ref,- valid_from_column=SCD2_VALID_FROM_COLUMN,- valid_to_column=SCD2_VALID_TO_COLUMN,+ delta_table_ref, valid_from_column=history.valid_from, valid_to_column=history.valid_to)delta_table = async_to_sync(scd2_writer.write)(data=pa_table,diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.pyindex 13f1504d0504..87e047858c38 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py@@ -20,8 +20,10 @@ from temporalio.service import RPCError, RPCStatusCodefrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (CDC_OP_COLUMN,CDC_SEQ_COLUMN,- SCD2_VALID_TO_COLUMN,+ HISTORY_NAMING,+ LEGACY_HISTORY_NAMING,TOAST_OMITTED_COLUMN,+ HistoryNaming,)from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.load import PostLoadResultfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY@@ -1737,7 +1739,8 @@ class TestEnrichCdcRows:assert result.column("name").to_pylist() == ["renamed", "two"]assert TOAST_OMITTED_COLUMN not in result.column_names- def test_scd2_fill_uses_current_row_not_history(self):+ @parameterized.expand([("legacy_table", LEGACY_HISTORY_NAMING), ("new_table", HISTORY_NAMING)])+ def test_scd2_fill_uses_current_row_not_history(self, _name: str, history: HistoryNaming) -> None:with tempfile.TemporaryDirectory() as path:ts = pa.timestamp("us", tz="UTC")write_deltalake(@@ -1747,13 +1750,18 @@ class TestEnrichCdcRows:"id": pa.array([1, 1], pa.int64()),"name": pa.array(["v1", "v2"], pa.string()),"big": pa.array(["old-toast", "current-toast"], pa.string()),- SCD2_VALID_TO_COLUMN: pa.array([1_000_000, None], ts),+ history.valid_from: pa.array([0, 1_000_000], ts),+ history.valid_to: pa.array([1_000_000, None], ts),}),mode="overwrite",)+ # The lane stamps the batch with its table's validity columns before the loader sees it.batch = self._batch([{"id": 1, "op": "U", "name": "v3", "omitted": ["big"]}])+ batch = batch.append_column(history.valid_from, pa.array([2_000_000], ts)).append_column(+ history.valid_to, pa.array([None], ts)+ )result = _enrich_cdc_rows(batch,primary_keys=["id"],diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.pyindex 3439a609ac59..0b58bd73de30 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py@@ -23,6 +23,11 @@ from products.warehouse_sources.backend.models.external_data_schema import ()from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import HISTORY_NAMING, LEGACY_HISTORY_NAMING+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (+ HistoryTableNameTakenError,+ history_naming,+)from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConventionfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import (build_table_name,@@ -375,6 +380,71 @@ class TestRegisterCdcCompanionTableStamp:assert history.active == "orders_cdc__query_a"assert history.active_job_id == str(job.id)+ @pytest.mark.parametrize(+ ("resource_name", "validity", "expected"),+ [+ pytest.param("orders_history", HISTORY_NAMING, HISTORY_NAMING, id="new_table"),+ pytest.param("orders_cdc", LEGACY_HISTORY_NAMING, LEGACY_HISTORY_NAMING, id="legacy_table"),+ ],+ )+ def test_the_next_run_reads_the_naming_the_table_was_created_with(self, team, resource_name, validity, expected):+ # Without the stamp, the next run would take its own `_history` table for another table's.+ source = ExternalDataSource.objects.create(+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=team, source_type="Postgres"+ )+ schema = ExternalDataSchema.objects.create(name="orders", team=team, source=source)+ job = ExternalDataJob.objects.create(+ team=team, pipeline=source, schema=schema, status=ExternalDataJobStatus.RUNNING, rows_synced=10+ )++ with (+ patch.object(DataWarehouseTable, "get_columns", return_value={}),+ patch.object(DataWarehouseTable, "get_count", return_value=100),+ ):+ async_to_sync(register_cdc_companion_table)(+ run_id=str(job.id),+ team_id=team.pk,+ schema_id=schema.id,+ resource_name=resource_name,+ row_count=100,+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,+ queryable_folder=f"{resource_name}__query_a",+ table_schema_dict={"id": "Int64", validity.valid_from: "DateTime", validity.valid_to: "DateTime"},+ )++ assert history_naming(ExternalDataSchema.objects.get(id=schema.id)) == expected++ def test_does_not_take_the_record_of_a_source_table_with_the_history_name(self, team):+ source = ExternalDataSource.objects.create(+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=team, source_type="Postgres"+ )+ schema = ExternalDataSchema.objects.create(name="orders", team=team, source=source)+ sibling_table = DataWarehouseTable.objects.create(+ team=team,+ name=build_table_name(source, "orders_cdc"),+ format=DataWarehouseTableFormat.DeltaS3Wrapper,+ url_pattern="s3://bucket/orders_cdc__query_a",+ external_data_source=source,+ )+ ExternalDataSchema.objects.create(name="orders_cdc", team=team, source=source, table=sibling_table)+ job = ExternalDataJob.objects.create(+ team=team, pipeline=source, schema=schema, status=ExternalDataJobStatus.RUNNING, rows_synced=10+ )++ with pytest.raises(HistoryTableNameTakenError):+ async_to_sync(register_cdc_companion_table)(+ run_id=str(job.id),+ team_id=team.pk,+ schema_id=schema.id,+ resource_name="orders_cdc",+ row_count=100,+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,+ queryable_folder="orders_cdc__query_b",+ )++ sibling_table.refresh_from_db()+ assert (sibling_table.url_pattern, sibling_table.is_history_table) == ("s3://bucket/orders_cdc__query_a", False)+def test_records_the_companion_size_on_the_companion_table(self, team):# In `both` mode the schema's table is the snapshot table. The companion size must land on# the companion's own record, for a new record and for an existing one.diff --git a/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py b/products/warehouse_sources/backend/temporal/data_imports/schema_flags.pyindex 92441c8bafdf..b612a0242387 100644--- a/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py+++ b/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py@@ -17,6 +17,7 @@ from posthog.temporal.common.utils import retry_on_db_connection_dropif TYPE_CHECKING:from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+ from products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcedef is_schema_flag_enabled(schema: ExternalDataSchema, flag: str) -> bool:@@ -26,10 +27,25 @@ def is_schema_flag_enabled(schema: ExternalDataSchema, flag: str) -> bool:released to a single table — set a release condition `schema_id = <id>` to dogfood a featureon one schema before rolling out by team/org/project."""+ return _is_team_flag_enabled(+ schema.team_id,+ flag,+ {"schema_id": str(schema.id), "team_id": str(schema.team_id), "source_type": schema.source.source_type},+ )+++def is_source_flag_enabled(source: ExternalDataSource, flag: str) -> bool:+ """Evaluate a rollout flag for a whole source, for a decision made before its schemas exist."""+ return _is_team_flag_enabled(+ source.team_id, flag, {"team_id": str(source.team_id), "source_type": source.source_type}+ )+++def _is_team_flag_enabled(team_id: int, flag: str, person_properties: dict[str, str]) -> bool:from posthog.models import Team # noqa: PLC0415 — keeps this module importable before app registry setuptry:- team = retry_on_db_connection_drop(lambda: Team.objects.only("uuid", "organization_id").get(id=schema.team_id))+ team = retry_on_db_connection_drop(lambda: Team.objects.only("uuid", "organization_id").get(id=team_id))except Team.DoesNotExist:return Falseexcept (OperationalError, InterfaceError) as e:@@ -45,11 +61,7 @@ def is_schema_flag_enabled(schema: ExternalDataSchema, flag: str) -> bool:flag,str(team.uuid),groups={"organization": str(team.organization_id), "project": str(team.id)},- person_properties={- "schema_id": str(schema.id),- "team_id": str(schema.team_id),- "source_type": schema.source.source_type,- },+ person_properties=person_properties,group_properties={"organization": {"id": str(team.organization_id)},"project": {"id": str(team.id)},diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex f57857ee3e23..36fda6b2a774 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -845,6 +845,10 @@ class PostgresSource(# parent) — deterministic, so don't retry. `XminUnsupportedError` matches once Temporal# wraps the failure; the message fragment matches the raw activity-level `str(e)`."XminUnsupportedError": None,+ "is already the name of another table of this source": (+ "The history table for this table would have the same name as another table of this source. "+ "Choose the consolidated table only for this table, or rename the other table on your database."+ ),"xmin replication": None,"NoSuchTableError": None,"is not permitted to log in": _ROLE_CANNOT_LOG_IN_ERROR,diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex d2620a3360b8..67bccb2a624c 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -63,6 +63,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.sharepointfrom products.warehouse_sources.backend.temporal.data_imports.sources.stripe.source import StripeSourcefrom products.warehouse_sources.backend.tests.api.utils import create_external_data_source_ok+_PH_HISTORY_NAMING_FLAG = (+ "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"+)+pytestmark = [pytest.mark.django_db,]@@ -1358,13 +1362,13 @@ class TestExternalDataSchema(APIBaseTest):{"primary_key_columns": ["id"],"cdc_table_mode": "both",- "schema_metadata": {"columns": [{"name": "id"}, {"name": "valid_from"}]},+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "_ph_valid_from"}]},},None,status.HTTP_400_BAD_REQUEST,ExternalDataSchema.SyncType.FULL_REFRESH,None,- "valid_from",+ "_ph_valid_from",),("accepts_a_validity_column_name_without_a_history_table",@@ -4599,7 +4603,6 @@ class TestSerializerLostUpdateProtection(APIBaseTest):[("turning_history_on_over_a_synced_column_is_refused",- "valid_to","consolidated",None,{"cdc_table_mode": "both"},@@ -4607,7 +4610,6 @@ class TestSerializerLostUpdateProtection(APIBaseTest):),("turning_history_on_over_a_column_left_out_is_allowed",- "valid_to","consolidated",["id"],{"cdc_table_mode": "both"},@@ -4615,52 +4617,75 @@ class TestSerializerLostUpdateProtection(APIBaseTest):),("adding_the_column_to_enabled_columns_is_refused",- "valid_to","both",["id"],{"enabled_columns": ["id", "valid_to"]},True,),+ ("editing_a_table_that_already_syncs_it_is_allowed", "both", None, {"enabled_columns": None}, False),(- "editing_a_table_that_already_syncs_it_is_allowed",+ "a_new_history_table_leaves_valid_to_to_the_source",+ "consolidated",+ None,+ {"cdc_table_mode": "both"},+ False,"valid_to",- "both",+ False,+ ),+ (+ "a_new_history_table_refuses_its_own_validity_name",+ "consolidated",None,- {"enabled_columns": None},+ {"cdc_table_mode": "both"},+ True,+ "_ph_valid_to",False,),# Capture columns are refused on every CDC table, history or not.("adding_a_capture_column_to_enabled_columns_is_refused",- "_PH_DELETED","consolidated",["id"],{"enabled_columns": ["id", "_PH_DELETED"]},True,+ "_PH_DELETED",+ False,),("editing_a_table_that_already_syncs_a_capture_column_is_allowed",- "_PH_DELETED","consolidated",None,{"enabled_columns": None},False,+ "_PH_DELETED",+ False,),])+ @mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)def test_patch_checks_the_columns_a_cdc_sync_writes(self,_name: str,- column: str,initial_mode: str,initial_enabled_columns: list[str] | None,request_body: dict,refused: bool,+ column: str = "valid_to",+ legacy_table: bool = True,):from products.warehouse_sources.backend.presentation.views.external_data_schema import (ExternalDataSchemaSerializer,)+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name+ if legacy_table:+ DataWarehouseTable.objects.create(+ team=self.team,+ name=build_table_name(self.source, f"{self.schema.name}_cdc"),+ format="DeltaS3Wrapper",+ url_pattern="s3://bucket/orders_cdc",+ external_data_source=self.source,+ )self.schema.sync_type_config["cdc_table_mode"] = initial_modeself.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": column}]}self.schema.enabled_columns = initial_enabled_columns@@ -4690,6 +4715,81 @@ class TestSerializerLostUpdateProtection(APIBaseTest):assert self.schema.enabled_columns == (initial_enabled_columns if refused else request_body.get("enabled_columns", initial_enabled_columns))+ keeps_history = not refused and self.schema.cdc_table_mode in ("cdc_only", "both")+ expected_pin = ("legacy" if legacy_table else "ph_validity") if keeps_history else None+ assert self.schema.sync_type_config.get("history_naming") == expected_pin++ def _save_through_serializer(self, schema: ExternalDataSchema, request_body: dict) -> None:+ from products.warehouse_sources.backend.presentation.views.external_data_schema import (+ ExternalDataSchemaSerializer,+ )++ serializer = ExternalDataSchemaSerializer(+ schema, data=request_body, partial=True, context={"team_id": self.team.pk, "post_commit_actions": []}+ )+ with mock.patch(+ "products.warehouse_sources.backend.presentation.views.external_data_schema.is_any_external_data_schema_paused",+ return_value=False,+ ):+ serializer.is_valid(raise_exception=True)+ serializer.save()++ @mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)+ def test_patch_refuses_a_history_table_named_like_a_synced_table(self):+ ExternalDataSchema.objects.create(+ name="public.orders_history", team=self.team, source=self.source, should_sync=True+ )++ with pytest.raises(ValidationError, match="share its name"):+ self._save_through_serializer(self.schema, {"cdc_table_mode": "both"})++ self.schema.refresh_from_db()+ assert self.schema.cdc_table_mode == "consolidated"++ @mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)+ def test_patch_refuses_syncing_a_table_named_like_another_schemas_history_table(self):+ # The history table is not registered until the first snapshot loads, and the name is taken already.+ self.schema.sync_type_config["cdc_table_mode"] = "both"+ self.schema.save()+ sibling = ExternalDataSchema.objects.create(+ name="public.orders_history",+ team=self.team,+ source=self.source,+ should_sync=False,+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,+ )++ with pytest.raises(ValidationError, match="history table of another table"):+ self._save_through_serializer(sibling, {"should_sync": True})++ sibling.refresh_from_db()+ assert sibling.should_sync is False++ @parameterized.expand(+ [+ ("no_history_table", ExternalDataSchema.SyncType.CDC, "consolidated", False, None),+ ("legacy_naming", ExternalDataSchema.SyncType.CDC, "both", False, "public.orders_cdc"),+ ("new_naming", ExternalDataSchema.SyncType.CDC, "both", True, "public.orders_history"),+ # Leaving CDC keeps the stored table mode, but the table no longer keeps history.+ ("left_cdc", ExternalDataSchema.SyncType.INCREMENTAL, "both", True, None),+ ]+ )+ def test_history_table_name_names_the_table_the_mode_writes(+ self, _name: str, sync_type: str, mode: str, ph_naming: bool, storage_name: str | None+ ):+ from products.warehouse_sources.backend.presentation.views.external_data_schema import (+ ExternalDataSchemaSerializer,+ )+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name++ self.schema.sync_type = sync_type+ self.schema.sync_type_config["cdc_table_mode"] = mode+ self.schema.save()++ with mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: ph_naming):+ name = ExternalDataSchemaSerializer().get_history_table_name(self.schema)++ assert name == (None if storage_name is None else build_table_name(self.source, storage_name))def test_patch_does_not_revert_a_concurrent_delete_data(self):from products.warehouse_sources.backend.presentation.views.external_data_schema import (diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex 56477cc0e485..a8d2add0c3cd 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -141,6 +141,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.soufrom products.warehouse_sources.backend.temporal.data_imports.sources.tiktok_ads.utils import TikTokAdsAPIErrorfrom products.warehouse_sources.backend.temporal.data_imports.sources.trino.source import TrinoSource+_PH_HISTORY_NAMING_FLAG = (+ "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"+)+def _configure_source_mock_versioning(mock_get_source) -> None:"""Tests that patch `SourceRegistry.get_source` with a bare MagicMock must give the versioning@@ -3404,6 +3408,7 @@ class TestExternalDataSource(APIBaseTest):"row_filter_columns": None,"available_columns": [],"source_column_metadata_available": False,+ "history_table_name": None,"source": None,"api_version": None,"api_version_deprecation": None,@@ -5004,6 +5009,7 @@ class TestExternalDataSource(APIBaseTest):assert enabled.sync_type == ExternalDataSchema.SyncType.CDCassert enabled.sync_type_config["cdc_mode"] == "snapshot"assert enabled.sync_type_config["cdc_table_mode"] == (events_table_mode or "consolidated")+ assert enabled.sync_type_config.get("history_naming") == ("legacy" if expected_history_modes else None)history_events = [c.kwargs["properties"]for c in mock_capture.call_args_list@@ -5027,7 +5033,7 @@ class TestExternalDataSource(APIBaseTest):@parameterized.expand([- ("no_primary_key", [("id", "uuid", False)], {}, "primary key", "consolidated", None),+ ("no_primary_key", [("id", "uuid", False)], {}, "primary key", "consolidated", None, None),("reserved_column",[("id", "uuid", False), ("_ph_cdc_seq", "bigint", True)],@@ -5035,6 +5041,7 @@ class TestExternalDataSource(APIBaseTest):"_ph_cdc_seq","consolidated",None,+ None,),("capture_column_the_load_renames",@@ -5043,22 +5050,34 @@ class TestExternalDataSource(APIBaseTest):"tracking_link (_ph_deleted)","consolidated",None,+ None,),("validity_column_with_a_history_table",- [("id", "uuid", False), ("valid_to", "timestamp", True)],+ [("id", "uuid", False), ("_ph_valid_to", "timestamp", True)],{"tracking_link": ["id"]},- "valid_to",+ "_ph_valid_to","cdc_only",None,+ None,),("validity_column_in_enabled_columns_that_hold_a_non_string",- [("id", "uuid", False), ("valid_to", "timestamp", True)],+ [("id", "uuid", False), ("_ph_valid_to", "timestamp", True)],{"tracking_link": ["id"]},- "valid_to",+ "_ph_valid_to","cdc_only",- ["id", "valid_to", {}],+ ["id", "_ph_valid_to", {}],+ None,+ ),+ (+ "history_table_named_like_another_synced_table",+ [("id", "uuid", False)],+ {"tracking_link": ["id"]},+ "same name",+ "both",+ None,+ "tracking_link_history",),])@@ -5075,6 +5094,7 @@ class TestExternalDataSource(APIBaseTest):@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.get_primary_key_columns")@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.cdc_pg_connection")@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.SourceRegistry.get_source")+ @patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)def test_create_postgres_cdc_rejects_a_table_it_cannot_capture(self,_name,@@ -5083,6 +5103,7 @@ class TestExternalDataSource(APIBaseTest):expected_in_message,cdc_table_mode,enabled_columns,+ sibling_table,mock_get_source,mock_cdc_pg_connection,mock_get_primary_key_columns,@@ -5119,7 +5140,23 @@ class TestExternalDataSource(APIBaseTest):foreign_keys=[],source_schema="public",source_table_name="tracking_link",- )+ ),+ *(+ [+ SourceSchema(+ name=sibling_table,+ supports_incremental=False,+ supports_append=False,+ supports_cdc=False,+ columns=[("id", "uuid", False)],+ foreign_keys=[],+ source_schema="public",+ source_table_name=sibling_table,+ )+ ]+ if sibling_table+ else []+ ),]mock_cdc_pg_connection.return_value.__enter__.return_value = object()@@ -5147,6 +5184,11 @@ class TestExternalDataSource(APIBaseTest):"cdc_table_mode": cdc_table_mode,**({"enabled_columns": enabled_columns} if enabled_columns is not None else {}),},+ *(+ [{"name": sibling_table, "should_sync": True, "sync_type": "full_refresh"}]+ if sibling_table+ else []+ ),],},},@@ -12088,23 +12130,38 @@ class TestDisableCDC(APIBaseTest):purged = {call.args[1] for call in mock_purge.call_args_list}assert purged == {str(cdc_schema.id)} # CDC schemas only+ @parameterized.expand(+ [+ # Registered before the history marker existed, so only its name says what it is.+ ("legacy_table_without_the_marker", "orders_cdc", {}),+ ("new_table", "orders_history", {"history_table": True, "history_naming": "ph_validity"}),+ ]+ )@patch("products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.cleanup_resources",return_value=None,)- def test_disable_cdc_soft_deletes_companion_cdc_tables(self, _cleanup) -> None:+ def test_disable_cdc_soft_deletes_companion_cdc_tables(+ self, _name: str, storage_name: str, options: dict, _cleanup+ ) -> None:+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name+source = _make_postgres_source(self.team.pk, self.user, cdc_enabled=True)+ ExternalDataSchema.objects.create(+ team_id=self.team.pk, name="orders", source=source, sync_type=ExternalDataSchema.SyncType.CDC+ )cdc_companion = DataWarehouseTable.objects.create(team_id=self.team.pk,- name="pg_orders_cdc",+ name=build_table_name(source, storage_name),external_data_source_id=source.pk,format=DataWarehouseTable.TableFormat.DeltaS3Wrapper,- url_pattern="s3://bucket/orders_cdc",+ url_pattern=f"s3://bucket/{storage_name}",+ options=options,)main_table = DataWarehouseTable.objects.create(team_id=self.team.pk,- name="pg_orders",+ name=build_table_name(source, "orders"),external_data_source_id=source.pk,format=DataWarehouseTable.TableFormat.DeltaS3Wrapper,url_pattern="s3://bucket/orders",diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.tsindex 7a0d245ef220..c4cba5e5575a 100644--- a/products/warehouse_sources/frontend/generated/api.schemas.ts+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts@@ -533,6 +533,11 @@ export interface ExternalDataSchemaApi {readonly available_columns: readonly ExternalDataSchemaApiAvailableColumnsItem[]/** Whether exact source-side column metadata is available for safe source-query projection. */readonly source_column_metadata_available: boolean+ /**+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.+ * @nullable+ */+ readonly history_table_name: string | null/*** Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.* @nullable@@ -723,6 +728,11 @@ export interface PatchedExternalDataSchemaApi {readonly available_columns?: readonly PatchedExternalDataSchemaApiAvailableColumnsItem[]/** Whether exact source-side column metadata is available for safe source-query projection. */readonly source_column_metadata_available?: boolean+ /**+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.+ * @nullable+ */+ readonly history_table_name?: string | null/*** Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.* @nullablediff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 303db48b942d..39f49a66d683 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -51824,6 +51824,11 @@ export namespace Schemas {readonly available_columns: readonly ExternalDataSchemaAvailableColumnsItem[];/** Whether exact source-side column metadata is available for safe source-query projection. */readonly source_column_metadata_available: boolean;+ /**+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.+ * @nullable+ */+ readonly history_table_name: string | null;/*** Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.* @nullable@@ -88018,6 +88023,11 @@ export namespace Schemas {readonly available_columns?: readonly PatchedExternalDataSchemaAvailableColumnsItem[];/** Whether exact source-side column metadata is available for safe source-query projection. */readonly source_column_metadata_available?: boolean;+ /**+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.+ * @nullable+ */+ readonly history_table_name?: string | null;/*** Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.* @nullable