Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): name new history tables _history with _ph_valid_* columns (#123551)

Daniel RCb1b8d259190c
diff --git a/products/data_warehouse/backend/presentation/views/table.py b/products/data_warehouse/backend/presentation/views/table.py
index f8ec43401630..8b4d2ff99e8b 100644
--- a/products/data_warehouse/backend/presentation/views/table.py
+++ b/products/data_warehouse/backend/presentation/views/table.py
@@ -692,11 +692,12 @@ class TableViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viewsets.M
old_csv_allow_double_quotes = instance.csv_allow_double_quotes
was_history_table = instance.is_history_table
+ had_ph_history_naming = instance.has_ph_history_naming
for attr, value in validated_data.items():
setattr(instance, attr, value)
- # The sync owns this marker, and an `options` edit through this API must not drop it.
+ # The sync owns these markers, and an `options` edit through this API must not drop them.
if was_history_table:
- instance.mark_as_history_table()
+ instance.mark_as_history_table(ph_naming=had_ph_history_naming)
if (
instance._is_csv_format()
diff --git a/products/desktop/packages/api-client/src/generated.ts b/products/desktop/packages/api-client/src/generated.ts
index bd9eee6a6413..3984d4d69833 100644
--- a/products/desktop/packages/api-client/src/generated.ts
+++ b/products/desktop/packages/api-client/src/generated.ts
@@ -11022,6 +11022,10 @@ export namespace Schemas {
* Whether exact source-side column metadata is available for safe source-query projection.
*/
source_column_metadata_available: boolean;
+ /**
+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.
+ */
+ history_table_name: string | null;
/**
* Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.
*/
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.py
index 2ca5d6b15764..b1614ef14001 100644
--- a/products/warehouse_sources/backend/facade/source_management.py
+++ b/products/warehouse_sources/backend/facade/source_management.py
@@ -31,6 +31,14 @@ _LAZY = {
"purge_buffer_prefix": "cdc.buffer",
"is_history_table_mode": "cdc.batcher",
"reserved_column_collisions": "cdc.batcher",
+ "history_naming_unless_taken": "cdc.history_table",
+ "pin_history_naming": "cdc.history_table",
+ "pinned_history_naming": "cdc.history_table",
+ "new_history_naming": "cdc.history_table",
+ "new_history_names_taken": "cdc.history_table",
+ "history_table_name": "cdc.history_table",
+ "history_table_ids": "cdc.history_table",
+ "is_another_schemas_history_table": "cdc.history_table",
"reserved_column_names": "cdc.batcher",
"BUFFER_LANE": "cdc.snapshot_lane",
"CDC_RESET_PENDING_KEY": "cdc.snapshot_lane",
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index 73758015baab..bb690d085a01 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -195,6 +195,11 @@ HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset({"_ph_cdc_op", "_ph_cdc_seq",
# Set in `options` when the sync registers a table as a schema's history table. Column names cannot
# tell the two apart, because a source table can have its own `valid_from` and `valid_to` columns.
HISTORY_TABLE_OPTION = "history_table"
+# Set in `options` when the sync creates a history table named `<table>_history` with the validity
+# columns `_ph_valid_from` / `_ph_valid_to`. A history table without it is older and keeps `_cdc`,
+# `valid_from` and `valid_to`.
+HISTORY_NAMING_OPTION = "history_naming"
+PH_HISTORY_NAMING = "ph_validity"
# Characters that carry quoting meaning once a column name is written into SQL. The structure
@@ -617,8 +622,16 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
def is_history_table(self) -> bool:
return (self.options or {}).get(HISTORY_TABLE_OPTION) is True
- def mark_as_history_table(self) -> None:
- self.options = {**(self.options or {}), HISTORY_TABLE_OPTION: True}
+ def mark_as_history_table(self, *, ph_naming: bool = False) -> None:
+ self.options = {
+ **(self.options or {}),
+ HISTORY_TABLE_OPTION: True,
+ **({HISTORY_NAMING_OPTION: PH_HISTORY_NAMING} if ph_naming else {}),
+ }
+
+ @property
+ def has_ph_history_naming(self) -> bool:
+ return (self.options or {}).get(HISTORY_NAMING_OPTION) == PH_HISTORY_NAMING
def hidden_columns(self) -> frozenset[str]:
"""The plumbing columns to keep out of the catalog for this table."""
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 067527870a35..882f0bd4744b 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -74,7 +74,13 @@ from products.warehouse_sources.backend.facade.source_management import (
filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,
get_cdc_adapter,
hand_reset_to_capture_if_sync_running,
+ history_naming_unless_taken,
+ history_table_name,
+ is_another_schemas_history_table,
is_history_table_mode,
+ new_history_naming,
+ pin_history_naming,
+ pinned_history_naming,
purge_buffer_prefix,
repair_is_running,
reserved_column_collisions,
@@ -466,14 +472,30 @@ def _refuse_reserved_columns(
new_mode: str | None,
new_enabled_columns: list[str] | None,
new_primary_keys: list[str] | None,
+ resulting_config: dict[str, Any],
) -> None:
"""Capture and the history table write their own columns next to the source's, so a synced source
column that loads under one of those names fails the sync. Only a collision this request adds is
- refused, so a table that already has one can still be edited or switched back."""
+ refused, so a table that already has one can still be edited or switched back.
+
+ Pins the naming the columns were checked against on `resulting_config`, so a later flag change cannot
+ refuse a column this request accepted."""
+ history = None
+ resolved = None
+ if is_history_table_mode(new_mode) or is_history_table_mode(previous_mode):
+ history = resolved = history_naming_unless_taken(instance)
+ kept_history = instance.sync_type == ExternalDataSchema.SyncType.CDC and is_history_table_mode(previous_mode)
+ if history is None and is_history_table_mode(new_mode) and not kept_history:
+ raise ValidationError(
+ f"A history table for {instance.name} would share its name with another table of this "
+ "source. Choose the consolidated table only for this table."
+ )
+ # A taken name leaves only the naming a new history table would get.
+ history = history or new_history_naming(instance.source)
columns = _source_column_names(instance)
taken = reserved_column_collisions(
columns,
- reserved_column_names(history=is_history_table_mode(new_mode)),
+ reserved_column_names(history if is_history_table_mode(new_mode) else None),
enabled_columns=new_enabled_columns,
primary_keys=new_primary_keys,
incremental_field=instance.incremental_field,
@@ -481,7 +503,7 @@ def _refuse_reserved_columns(
if taken and instance.sync_type == ExternalDataSchema.SyncType.CDC:
existing = reserved_column_collisions(
columns,
- reserved_column_names(history=is_history_table_mode(previous_mode)),
+ reserved_column_names(history if is_history_table_mode(previous_mode) else None),
enabled_columns=previous_enabled_columns,
primary_keys=previous_primary_keys,
incremental_field=instance.incremental_field,
@@ -493,6 +515,8 @@ def _refuse_reserved_columns(
"PostHog writes. Rename the column on your database, leave it out of the synced columns, or change "
"this table's sync method or output tables."
)
+ if resolved is not None and is_history_table_mode(new_mode) and pinned_history_naming(instance) is None:
+ pin_history_naming(resulting_config, resolved)
def schema_display_status(schema: ExternalDataSchema) -> str | None:
@@ -686,6 +710,11 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
read_only=True,
help_text="Whether exact source-side column metadata is available for safe source-query projection.",
)
+ history_table_name = serializers.SerializerMethodField(
+ read_only=True,
+ help_text="Name of the history table that keeps every version of each row, or `null` when the table "
+ "keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.",
+ )
# `source` shadows DRF's reserved `Field.source` attribute, so mypy flags the assignment;
# the runtime behaviour (a read-only SerializerMethodField backed by get_source) is correct.
source = serializers.SerializerMethodField( # type: ignore[assignment]
@@ -726,6 +755,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"row_filter_columns",
"available_columns",
"source_column_metadata_available",
+ "history_table_name",
"source",
"api_version",
"api_version_deprecation",
@@ -746,11 +776,16 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
"row_filter_columns",
"available_columns",
"source_column_metadata_available",
+ "history_table_name",
"source",
"api_version_deprecation",
"user_access_level",
]
+ @extend_schema_field(serializers.CharField(allow_null=True))
+ def get_history_table_name(self, schema: ExternalDataSchema) -> str | None:
+ return history_table_name(schema)
+
@extend_schema_field(
{
"type": "array",
@@ -1374,6 +1409,20 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
new_mode=resulting_config.get("cdc_table_mode"),
new_enabled_columns=validated_data.get("enabled_columns", instance.enabled_columns),
new_primary_keys=resulting_config.get("primary_key_columns"),
+ resulting_config=resulting_config,
+ )
+ instance.sync_type_config = resulting_config
+ validated_data["sync_type_config"] = resulting_config
+
+ # Its table and the history table would share one name and one storage folder.
+ if (
+ validated_data.get("should_sync")
+ and not instance.should_sync
+ and is_another_schemas_history_table(instance)
+ ):
+ raise ValidationError(
+ f"{instance.name} has the same name as the history table of another table of this source, so it "
+ "can't sync. Rename the table on your database to sync it."
)
should_sync = validated_data.get("should_sync", None)
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
index d78078f82ddb..f569ef03f478 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
@@ -36,6 +36,7 @@ from products.warehouse_sources.backend.facade.source_management import (
CDCSourceAdapter,
PostgresSource,
get_cdc_adapter,
+ history_table_ids,
repair_cdc_source,
source_type_supports_cdc,
)
@@ -535,20 +536,16 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
except ExternalDataSchema.DoesNotExist:
pass
+ table_ids = history_table_ids(instance)
+
# Force CDC schemas to pick a new strategy by clearing sync_type and pausing.
ExternalDataSchema.objects.filter(
source=instance,
sync_type=ExternalDataSchema.SyncType.CDC,
).exclude(deleted=True).update(sync_type=None, should_sync=False)
- # Soft-delete `_cdc` companion DataWarehouseTable rows so the next sync
- # rebuilds them once the user picks a new strategy.
- DataWarehouseTable.objects.filter(
- external_data_source_id=instance.id,
- team_id=self.team_id,
- deleted=False,
- name__endswith="_cdc",
- ).update(deleted=True)
+ # Soft-delete the history tables so the next sync rebuilds them once the user picks a new strategy.
+ DataWarehouseTable.objects.filter(id__in=table_ids, team_id=self.team_id).update(deleted=True)
# Clear ALL cdc_* keys from job_inputs — leaving stale engine identifiers
# behind (e.g. `cdc_consistent_point`) would corrupt resume tracking if
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index 7869fab3340b..426b96e8da73 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -73,7 +73,10 @@ from products.warehouse_sources.backend.facade.source_management import (
get_cdc_adapter,
is_history_table_mode,
is_team_allowlisted_for_internal_hosts,
+ new_history_names_taken,
+ new_history_naming,
new_source_requires_ssl,
+ pin_history_naming,
reserved_column_collisions,
reserved_column_names,
sql_schema_metadata,
@@ -1390,6 +1393,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
# Pre-fetch PK column names for CDC tables
pk_columns_by_table: dict[str, list[str]] = {}
+ history_naming_config_by_schema: dict[str, dict[str, Any]] = {}
if cdc_enabled:
cdc_table_names_by_schema: dict[str, set[str]] = {}
cdc_schema_name_by_location: dict[tuple[str, str], str] = {}
@@ -1457,10 +1461,11 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
# Capture and the history table write their own columns next to the source's, so a synced
# source column that loads under one of those names would fail the source's first sync.
# Refuse before any replication state exists.
+ new_history = new_history_naming(new_source_model)
reserved_columns_by_table = {
schema["name"]: reserved_column_collisions(
[column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],
- reserved_column_names(history=is_history_table_mode(schema.get("cdc_table_mode"))),
+ reserved_column_names(new_history if is_history_table_mode(schema.get("cdc_table_mode")) else None),
enabled_columns=_payload_enabled_columns(schema),
primary_keys=pk_columns_by_table.get(schema["name"]),
)
@@ -1485,6 +1490,37 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
},
)
+ synced_names = [
+ schema["name"]
+ for schema in payload_schemas
+ if schema.get("should_sync", False) and isinstance(schema.get("name"), str)
+ ]
+ history_names = [
+ schema["name"]
+ for schema in payload_schemas
+ if schema.get("sync_type") == "cdc"
+ and schema.get("should_sync", False)
+ and is_history_table_mode(schema.get("cdc_table_mode"))
+ and isinstance(schema.get("name"), str)
+ ]
+ tables_with_taken_history_name = new_history_names_taken(
+ new_source_model, new_history, history_names, synced_names
+ )
+ if tables_with_taken_history_name:
+ new_source_model.delete()
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={
+ "message": (
+ "A history table would have the same name as another table you chose to sync. Choose "
+ "the consolidated table only, or leave the other table out, for these tables: "
+ f"{', '.join(tables_with_taken_history_name)}."
+ )
+ },
+ )
+ for name in history_names:
+ pin_history_naming(history_naming_config_by_schema.setdefault(name, {}), new_history)
+
# Engine-side CDC resource setup runs after PK validation so we don't leave
# replication state on the source for a config we're about to refuse.
if cdc_enabled:
@@ -1699,6 +1735,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
"schema_metadata": schema_metadata,
"cdc_table_mode": cdc_table_mode,
}
+ sync_type_config.update(history_naming_config_by_schema.get(schema_name, {}))
else:
sync_type_config = {"schema_metadata": schema_metadata}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
index d79c5e7e0d08..7949de648778 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
@@ -46,10 +46,12 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters impor
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_SEQ_COLUMN,
- HISTORY_COLUMNS,
+ HISTORY_NAMING,
ChangeEventBatcher,
+ HistoryNaming,
enrich_delete_rows,
enrich_toast_omitted_rows,
+ history_columns,
is_history_table_mode,
reserved_column_collisions,
)
@@ -75,6 +77,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import
CDCSlotNotConfiguredError,
classify_cdc_error,
)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import SourceHistoryNamings
from products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import has_engine_seq
from products.warehouse_sources.backend.temporal.data_imports.cdc.naming import (
CDC_EXTRACTION_WORKFLOW_ID_PREFIX,
@@ -203,6 +206,7 @@ class CDCExtractActivity:
self.split_key_columns: dict[str, list[str]] = {}
# Missing entry = sync all columns; otherwise the set is the projection (always includes PKs).
self.enabled_columns_by_table: dict[str, set[str]] = {}
+ self.history_naming_by_table: dict[str, HistoryNaming] = {}
self.adapter: typing.Any = None
self.reader: typing.Any = None
self.batcher: ChangeEventBatcher | None = None
@@ -381,8 +385,9 @@ class CDCExtractActivity:
# The batcher already refused the capture columns. The history table's own columns are
# checked here, because only the schema says whether this table keeps history.
- if is_history_table_mode(schema.cdc_table_mode):
- taken = reserved_column_collisions(raw_table.column_names, HISTORY_COLUMNS)
+ history = self.history_naming_by_table.get(table_name)
+ if history is not None:
+ taken = reserved_column_collisions(raw_table.column_names, history_columns(history))
if taken:
raise CDCReservedColumnError(f"Table {table_name} has history table column(s) {taken}")
@@ -615,7 +620,17 @@ class CDCExtractActivity:
}
)
+ history_namings: SourceHistoryNamings | None = None
for schema in self.cdc_schemas:
+ if is_history_table_mode(schema.cdc_table_mode):
+ if history_namings is None:
+ assert self.source is not None
+ history_namings = SourceHistoryNamings(self.source)
+ # A taken name fails this table's own import. Capture then refuses only the names every
+ # history table reserves, so the source's other tables keep syncing.
+ self.history_naming_by_table[schema.name] = (
+ history_namings.naming_unless_taken(schema) or HISTORY_NAMING
+ )
enabled = schema.enabled_columns
# `None` = sync all; `[]` = retain PKs + incremental only. Match the
# invariant used by build_select_clause / pipeline_sync / filter_dwh_columns.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
index 2646fe7f0d96..ff0d01cc2fd6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
@@ -5,6 +5,8 @@ from collections.abc import Callable, Iterable
import pyarrow as pa
+from posthog.dataclasses import frozen
+
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnError
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
@@ -24,18 +26,45 @@ CDC_SEQ_COLUMN = "_ph_cdc_seq"
# name.
CDC_SEQ_PROVENANCE = {b"posthog_cdc": b"engine_position"}
-# Suffix of the SCD2 companion table's resource name ({schema.name}_cdc).
-CDC_COMPANION_SUFFIX = "_cdc"
+
+@frozen
+class HistoryNaming:
+ """The table name suffix and the validity column names of one history table.
+
+ A history table keeps the naming it was created with, so code reads it from the table and never
+ from a constant.
+ """
+
+ suffix: str
+ valid_from: str
+ valid_to: str
+
+ @property
+ def validity_columns(self) -> frozenset[str]:
+ return frozenset({self.valid_from, self.valid_to})
+
+
+LEGACY_HISTORY_NAMING = HistoryNaming(suffix="_cdc", valid_from="valid_from", valid_to="valid_to")
+# The `_ph_` names leave a source's own `valid_from` and `valid_to` free to sync.
+HISTORY_NAMING = HistoryNaming(suffix="_history", valid_from="_ph_valid_from", valid_to="_ph_valid_to")
+
+
+def history_naming_of(column_names: Iterable[str]) -> HistoryNaming:
+ """The naming a stamped history batch or table was written with.
+
+ `_ph_valid_from` is reserved on every history table, so only the stamp can put it there.
+ """
+ return HISTORY_NAMING if HISTORY_NAMING.valid_from in set(column_names) else LEGACY_HISTORY_NAMING
-def companion_resource_name(schema_name: str) -> str:
- """Storage name for a schema's `_cdc` companion table.
+def companion_resource_name(schema_name: str, naming: HistoryNaming) -> str:
+ """Storage name for a schema's history table.
Keyed on the schema's `name`, never its resolved folder: the companion is CDC-only and stays
self-consistent with its `name`-keyed snapshot seed. Capture, the snapshot seed and the
buffered consumer all write the same table, so they all resolve the name here.
"""
- return f"{schema_name}{CDC_COMPANION_SUFFIX}"
+ return f"{schema_name}{naming.suffix}"
# Per-row list of source columns the change stream omitted because they are
@@ -44,10 +73,6 @@ def companion_resource_name(schema_name: str) -> str:
# writing to DeltaLake.
TOAST_OMITTED_COLUMN = "_ph_toast_omitted"
-# SCD Type 2 columns added to the _cdc history table
-SCD2_VALID_FROM_COLUMN = "valid_from"
-SCD2_VALID_TO_COLUMN = "valid_to"
-
# Values of `cdc_table_mode` that write the history table.
CDC_HISTORY_TABLE_MODES: frozenset[str] = frozenset({"cdc_only", "both"})
@@ -61,13 +86,20 @@ def is_history_table_mode(cdc_table_mode: object) -> bool:
CAPTURE_COLUMNS: frozenset[str] = frozenset(
{CDC_OP_COLUMN, CDC_TIMESTAMP_COLUMN, CDC_SEQ_COLUMN, DELETED_COLUMN, DELETED_AT_COLUMN, TOAST_OMITTED_COLUMN}
)
-# The columns that only the history table adds.
-HISTORY_COLUMNS: frozenset[str] = frozenset({SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN})
-def reserved_column_names(*, history: bool) -> frozenset[str]:
- """Every name a CDC sync writes next to the source's own columns."""
- return CAPTURE_COLUMNS | HISTORY_COLUMNS if history else CAPTURE_COLUMNS
+def history_columns(naming: HistoryNaming) -> frozenset[str]:
+ """The columns a history table adds to the capture columns.
+
+ `_ph_valid_*` stays reserved on a legacy history table too, because `history_naming_of` reads the
+ naming from it.
+ """
+ return HISTORY_NAMING.validity_columns | naming.validity_columns
+
+
+def reserved_column_names(history: HistoryNaming | None) -> frozenset[str]:
+ """Every name a CDC sync writes next to the source's own columns, `history` when it keeps a history table."""
+ return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history_columns(history)
def reserved_column_collisions(
@@ -89,20 +121,13 @@ def reserved_column_collisions(
return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in reserved)
-# Columns that are CDC/SCD2 metadata — always filled from the event itself,
-# never overwritten when enriching DELETE rows with last-known source data.
-_CDC_METADATA_COLUMNS: frozenset[str] = frozenset(
- {
- CDC_OP_COLUMN,
- CDC_TIMESTAMP_COLUMN,
- CDC_SEQ_COLUMN,
- DELETED_COLUMN,
- DELETED_AT_COLUMN,
- TOAST_OMITTED_COLUMN,
- SCD2_VALID_FROM_COLUMN,
- SCD2_VALID_TO_COLUMN,
- }
-)
+def metadata_columns(history: HistoryNaming | None) -> frozenset[str]:
+ """Columns always filled from the change event itself, never from the last known row.
+
+ `history` names the validity columns stamped on a history batch. A source column that only shares a
+ validity name with another naming is data.
+ """
+ return CAPTURE_COLUMNS if history is None else CAPTURE_COLUMNS | history.validity_columns
# Micro-batch thresholds for CDC WAL processing. Flushing periodically
@@ -230,6 +255,7 @@ def enrich_delete_rows(
table: pa.Table,
pk_columns: list[str],
existing_rows: pa.Table | None = None,
+ history: HistoryNaming | None = None,
) -> pa.Table:
"""Fill data columns on DELETE rows from the last known state.
@@ -246,8 +272,10 @@ def enrich_delete_rows(
The processor pre-filters `existing_rows` to only the PKs present in DELETE
events before calling this, so the materialization is bounded by the batch size.
- Metadata columns (op, timestamp, deleted flags, valid_from/to) are always
- kept from the DELETE event itself and are never overwritten.
+ Metadata columns (op, timestamp, deleted flags, and the validity columns of
+ `history` when the table is a stamped history batch) are always kept from the
+ DELETE event itself and are never overwritten. A source column that only shares
+ a validity name with another naming is data.
"""
if not pk_columns or table.num_rows == 0:
return table
@@ -262,7 +290,8 @@ def enrich_delete_rows(
return table
pk_set = set(present_pks)
- table_data_cols = [col for col in table.column_names if col not in _CDC_METADATA_COLUMNS and col not in pk_set]
+ metadata = metadata_columns(history)
+ table_data_cols = [col for col in table.column_names if col not in metadata and col not in pk_set]
# Determine extra columns that exist in existing_rows but not in the current table.
# A standalone DELETE event may only carry PK columns — we add the missing data
@@ -272,7 +301,7 @@ def enrich_delete_rows(
extra_cols_from_existing = [
col
for col in existing_rows.column_names
- if col not in _CDC_METADATA_COLUMNS and col not in pk_set and col not in table.column_names
+ if col not in metadata and col not in pk_set and col not in table.column_names
]
all_data_cols = table_data_cols + extra_cols_from_existing
@@ -461,8 +490,8 @@ def enrich_toast_omitted_rows(
return pa.table(new_columns, schema=pa.schema(new_fields))
-def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:
- """Add SCD Type 2 columns (valid_from, valid_to) to a raw CDC event table.
+def build_scd2_table(pa_table: pa.Table, pk_columns: list[str], naming: HistoryNaming) -> pa.Table:
+ """Add the SCD Type 2 columns of `naming` (valid_from, valid_to) to a raw CDC event table.
valid_from = _ph_cdc_timestamp (the commit timestamp of this event)
valid_to = _ph_cdc_timestamp of the next event for the same PK, or null
@@ -475,7 +504,7 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:
A two-step merge + append in the load processor closes previous "current"
rows (sets valid_to) when a new batch is written for the same PK.
"""
- taken = reserved_column_collisions(pa_table.column_names, HISTORY_COLUMNS)
+ taken = reserved_column_collisions(pa_table.column_names, history_columns(naming))
if taken:
# Delta refuses a duplicate column name at write time; failing here names the column and
# keeps a half-built batch out of the writer.
@@ -489,9 +518,9 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:
)
if pa_table.num_rows == 0:
- return pa_table.append_column(
- pa.field(SCD2_VALID_FROM_COLUMN, ts_type), pa.array([], type=ts_type)
- ).append_column(pa.field(SCD2_VALID_TO_COLUMN, ts_type), pa.array([], type=ts_type))
+ return pa_table.append_column(pa.field(naming.valid_from, ts_type), pa.array([], type=ts_type)).append_column(
+ pa.field(naming.valid_to, ts_type), pa.array([], type=ts_type)
+ )
ts_col = pa_table.column(CDC_TIMESTAMP_COLUMN).to_pylist()
@@ -515,10 +544,10 @@ def build_scd2_table(pa_table: pa.Table, pk_columns: list[str]) -> pa.Table:
valid_to[idx] = ts_col[next_idx]
return pa_table.append_column(
- pa.field(SCD2_VALID_FROM_COLUMN, ts_type),
+ pa.field(naming.valid_from, ts_type),
pa_table.column(CDC_TIMESTAMP_COLUMN),
).append_column(
- pa.field(SCD2_VALID_TO_COLUMN, ts_type),
+ pa.field(naming.valid_to, ts_type),
pa.array(valid_to, type=ts_type),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
index 793fce39f5d3..45e140eb5a18 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/errors.py
@@ -174,7 +174,7 @@ _CATEGORY_DEFAULTS: dict[CDCErrorCategory, tuple[str, bool]] = {
),
CDCErrorCategory.RESERVED_COLUMN_CONFLICT: (
"A source table has a column that loads under a name PostHog writes for change data capture, "
- "such as _ph_cdc_seq or _ph_deleted, or valid_from or valid_to on a table with a history table. "
+ "such as _ph_cdc_seq or _ph_valid_from, or valid_from on an older history table that ends in _cdc. "
"Rename the column on your database, leave it out of the synced columns, or choose another sync "
"method for that table.",
False,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/history_table.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/history_table.py
new file mode 100644
index 000000000000..b4124efc2845
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/history_table.py
@@ -0,0 +1,276 @@
+"""Find a schema's history table and the naming it was created with."""
+
+from __future__ import annotations
+
+from collections import defaultdict
+from collections.abc import Iterable
+from typing import Any
+from uuid import UUID
+
+from django.db.models import Q, QuerySet
+
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.models.table import HISTORY_NAMING_OPTION, PH_HISTORY_NAMING, DataWarehouseTable
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
+ HistoryNaming,
+ companion_resource_name,
+ is_history_table_mode,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import (
+ build_table_name,
+ resolve_table_and_folder_names,
+)
+from products.warehouse_sources.backend.temporal.data_imports.schema_flags import is_source_flag_enabled
+
+# New history tables are named `<table>_history` with `_ph_valid_from` / `_ph_valid_to` while this flag is on
+# for the source. Keep it off until every loader reads the validity names from the batch: an older loader
+# appends such a batch without closing the open versions.
+PH_HISTORY_NAMING_FLAG = "warehouse-ph-history-naming"
+_LEGACY_HISTORY_NAMING = "legacy"
+
+
+class HistoryTableNameTakenError(Exception):
+ """Another table of the source already has the catalog name a new history table would take.
+
+ Two registrations under one name hide one of the tables from queries, and the companion registration
+ would take the other table's record.
+ """
+
+
+def not_another_schemas_table(tables: QuerySet[DataWarehouseTable], schema_id: UUID) -> QuerySet[DataWarehouseTable]:
+ """Drop tables another schema syncs into.
+
+ A source table can carry a history table's name, as `orders_cdc` next to `orders` does. A history
+ table is linked to no schema, or to its own schema in `cdc_only` mode.
+ """
+ return tables.filter(Q(externaldataschema__isnull=True) | Q(externaldataschema__id=schema_id)).distinct()
+
+
+def new_history_naming(source: ExternalDataSource) -> HistoryNaming:
+ """The naming a history table of this source created now gets."""
+ return HISTORY_NAMING if is_source_flag_enabled(source, PH_HISTORY_NAMING_FLAG) else LEGACY_HISTORY_NAMING
+
+
+def pin_history_naming(sync_type_config: dict[str, Any], naming: HistoryNaming) -> None:
+ """Record on the schema the naming its columns were checked against.
+
+ The flag can change, or fail to evaluate, before the history table is registered. Capture would then
+ refuse a source column the API accepted.
+ """
+ sync_type_config[HISTORY_NAMING_OPTION] = PH_HISTORY_NAMING if naming == HISTORY_NAMING else _LEGACY_HISTORY_NAMING
+
+
+def pinned_history_naming(schema: ExternalDataSchema) -> HistoryNaming | None:
+ value = (schema.sync_type_config or {}).get(HISTORY_NAMING_OPTION)
+ if value == PH_HISTORY_NAMING:
+ return HISTORY_NAMING
+ return LEGACY_HISTORY_NAMING if value == _LEGACY_HISTORY_NAMING else None
+
+
+def kept_history_naming(schema: ExternalDataSchema) -> HistoryNaming | None:
+ """The naming of the history table the schema's mode writes, or None when it writes none."""
+ return history_naming(schema) if is_history_table_mode(schema.cdc_table_mode) else None
+
+
+def registered_history_naming(schema: ExternalDataSchema) -> HistoryNaming | None:
+ """The naming of the schema's registered history table, or None when it has none yet.
+
+ A stamped `_history` table wins over a table under the legacy name, which a source table can also
+ carry. A legacy table predates the stamp and the marker, so its name is all it has.
+ """
+ source = schema.source
+ names = {
+ build_table_name(source, companion_resource_name(schema.name, naming)): naming
+ for naming in (HISTORY_NAMING, LEGACY_HISTORY_NAMING)
+ }
+ tables = not_another_schemas_table(
+ DataWarehouseTable.objects.filter(
+ team_id=schema.team_id, external_data_source_id=source.id, deleted=False, name__in=names
+ ),
+ schema.id,
+ )
+ return _naming_of_registered({names[table.name]: table for table in tables})
+
+
+def _naming_of_registered(tables_by_naming: dict[HistoryNaming, DataWarehouseTable]) -> HistoryNaming | None:
+ current = tables_by_naming.get(HISTORY_NAMING)
+ if current is not None and current.has_ph_history_naming:
+ return HISTORY_NAMING
+ return LEGACY_HISTORY_NAMING if LEGACY_HISTORY_NAMING in tables_by_naming else None
+
+
+class SourceHistoryNamings:
+ """Resolves the history naming of a source's schemas from one read of its schemas and tables."""
+
+ def __init__(self, source: ExternalDataSource) -> None:
+ self._source = source
+ schemas = list(ExternalDataSchema.objects.filter(team_id=source.team_id, source_id=source.id, deleted=False))
+ self._schema_ids_by_table: defaultdict[UUID, set[UUID]] = defaultdict(set)
+ # A sibling that neither syncs nor holds a table registers nothing under its name. Turning its sync on
+ # later is refused by `is_another_schemas_history_table`.
+ self._synced_schema_ids_by_name: defaultdict[str, set[UUID]] = defaultdict(set)
+ for schema in schemas:
+ if schema.table_id is not None:
+ self._schema_ids_by_table[schema.table_id].add(schema.id)
+ if schema.should_sync or schema.table_id is not None:
+ names = resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name)
+ self._synced_schema_ids_by_name[build_table_name(source, names.table_storage_name)].add(schema.id)
+ self._tables_by_name: defaultdict[str, list[DataWarehouseTable]] = defaultdict(list)
+ for table in DataWarehouseTable.objects.filter(
+ team_id=source.team_id, external_data_source_id=source.id, deleted=False
+ ):
+ self._tables_by_name[table.name].append(table)
+ self._new_naming: HistoryNaming | None = None
+
+ def naming(self, schema: ExternalDataSchema) -> HistoryNaming:
+ """The naming of the schema's history table: its registration, else its pin, else `new_history_naming`.
+
+ Raises `HistoryTableNameTakenError` when a new history table would take the catalog name of another
+ table of the source.
+ """
+ registered = self._registered(schema)
+ if registered is not None:
+ return registered
+ naming = pinned_history_naming(schema) or self._naming_for_new_tables()
+ new_name = build_table_name(self._source, companion_resource_name(schema.name, naming))
+ if self._tables_by_name.get(new_name) or self._synced_schema_ids_by_name.get(new_name, set()) - {schema.id}:
+ raise HistoryTableNameTakenError(f"{new_name} is already the name of another table of this source")
+ return naming
+
+ def naming_unless_taken(self, schema: ExternalDataSchema) -> HistoryNaming | None:
+ try:
+ return self.naming(schema)
+ except HistoryTableNameTakenError:
+ return None
+
+ def _registered(self, schema: ExternalDataSchema) -> HistoryNaming | None:
+ # A table another schema syncs into is that schema's own, as `orders_cdc` next to `orders` is.
+ tables_by_naming = {
+ naming: table
+ for naming in (HISTORY_NAMING, LEGACY_HISTORY_NAMING)
+ for table in self._tables_by_name.get(
+ build_table_name(self._source, companion_resource_name(schema.name, naming)), []
+ )
+ if self._schema_ids_by_table.get(table.id, set()) <= {schema.id}
+ }
+ return _naming_of_registered(tables_by_naming)
+
+ def _naming_for_new_tables(self) -> HistoryNaming:
+ if self._new_naming is None:
+ self._new_naming = new_history_naming(self._source)
+ return self._new_naming
+
+
+def history_naming(schema: ExternalDataSchema) -> HistoryNaming:
+ """`SourceHistoryNamings.naming` for one schema."""
+ return SourceHistoryNamings(schema.source).naming(schema)
+
+
+def new_history_names_taken(
+ source: ExternalDataSource,
+ naming: HistoryNaming,
+ history_schema_names: Iterable[str],
+ synced_schema_names: Iterable[str],
+) -> list[str]:
+ """The schemas of a new source whose history table would take the catalog name of a synced schema."""
+ taken = {build_table_name(source, name) for name in synced_schema_names}
+ return sorted(
+ name
+ for name in history_schema_names
+ if build_table_name(source, companion_resource_name(name, naming)) in taken
+ )
+
+
+def history_naming_unless_taken(schema: ExternalDataSchema) -> HistoryNaming | None:
+ """`history_naming`, or None when the new name belongs to another table of the source."""
+ try:
+ return history_naming(schema)
+ except HistoryTableNameTakenError:
+ return None
+
+
+def is_another_schemas_history_table(schema: ExternalDataSchema) -> bool:
+ """Whether the schema's own table name belongs to the history table of another schema.
+
+ A sibling that keeps history counts before its history table is registered, which happens only after
+ its first snapshot loads.
+ """
+ source = schema.source
+ name = build_table_name(
+ source, resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name).table_storage_name
+ )
+ siblings = ExternalDataSchema.objects.filter(
+ team_id=schema.team_id, source_id=schema.source_id, deleted=False, sync_type=ExternalDataSchema.SyncType.CDC
+ ).exclude(id=schema.id)
+ # Only a sibling whose history name under some naming matches pays for the registration lookup.
+ candidates = [
+ sibling
+ for sibling in siblings
+ if any(
+ build_table_name(source, companion_resource_name(sibling.name, naming)) == name
+ for naming in (HISTORY_NAMING, LEGACY_HISTORY_NAMING)
+ )
+ ]
+ if any(history_table_name(sibling) == name for sibling in candidates):
+ return True
+ taken = (
+ DataWarehouseTable.objects.filter(
+ team_id=schema.team_id, external_data_source_id=schema.source_id, deleted=False, name=name
+ )
+ .exclude(externaldataschema__id=schema.id)
+ .first()
+ )
+ if taken is None:
+ return False
+ return taken.is_history_table or name in _legacy_history_names(source, exclude_schema_id=schema.id)
+
+
+def _schema_names(source: ExternalDataSource, *, exclude_schema_id: UUID | None = None) -> list[str]:
+ schemas = ExternalDataSchema.objects.filter(team_id=source.team_id, source_id=source.id, deleted=False)
+ if exclude_schema_id is not None:
+ schemas = schemas.exclude(id=exclude_schema_id)
+ return list(schemas.values_list("name", flat=True))
+
+
+def _legacy_history_names(source: ExternalDataSource, *, exclude_schema_id: UUID | None = None) -> set[str]:
+ return {
+ build_table_name(source, companion_resource_name(name, LEGACY_HISTORY_NAMING))
+ for name in _schema_names(source, exclude_schema_id=exclude_schema_id)
+ }
+
+
+def history_tables(source: ExternalDataSource) -> list[DataWarehouseTable]:
+ """The source's registered history tables.
+
+ One that stopped syncing before the marker existed, such as one whose schema dropped its history
+ table, carries only the legacy name of its schema. A table under a legacy name that a schema of that
+ very name syncs into, as `orders_cdc` next to `orders` does, is that schema's own table.
+ """
+ candidates = DataWarehouseTable.objects.filter(
+ team_id=source.team_id, external_data_source_id=source.id, deleted=False
+ ).filter(Q(options__history_table=True) | Q(name__in=_legacy_history_names(source)))
+ own_tables = {
+ table_id: build_table_name(source, name)
+ for table_id, name in ExternalDataSchema.objects.filter(
+ team_id=source.team_id, source_id=source.id, deleted=False, table_id__isnull=False
+ ).values_list("table_id", "name")
+ }
+ return [table for table in candidates if table.is_history_table or own_tables.get(table.id) != table.name]
+
+
+def history_table_ids(source: ExternalDataSource) -> list[UUID]:
+ return [table.id for table in history_tables(source)]
+
+
+def history_table_name(schema: ExternalDataSchema) -> str | None:
+ """The name of the history table the schema's mode writes, or None when it writes none.
+
+ Skips the name check of `history_naming`, so a list of schemas costs one query per history table.
+ """
+ if not schema.keeps_history:
+ return None
+ naming = registered_history_naming(schema) or pinned_history_naming(schema) or new_history_naming(schema.source)
+ return build_table_name(schema.source, companion_resource_name(schema.name, naming))
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.py
index db4477c8a334..e70d07ef751b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/lane_position.py
@@ -15,7 +15,7 @@ from __future__ import annotations
import asyncio
from collections import Counter, defaultdict
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
from functools import partial
from typing import TYPE_CHECKING, Any, cast
from urllib.parse import unquote
@@ -29,8 +29,9 @@ from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_SEQ_COLUMN,
- SCD2_VALID_FROM_COLUMN,
- SCD2_VALID_TO_COLUMN,
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
+ history_naming_of,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrow_utils import normalize_column_name
@@ -91,6 +92,7 @@ class LanePosition:
# False above the position-row cap: `applied` then carries keys only, and a batch row is
# matched on key and operation alone.
content_matched: bool = True
+ non_content_columns: frozenset[str] = LEGACY_HISTORY_NAMING.validity_columns
# Reads the rows at the position into a resolved copy of this. Deferred so a tick whose
# buffer holds nothing at the position — every idle tick — never reads them at all.
load_applied: Callable[[], LanePosition] | None = None
@@ -98,13 +100,21 @@ class LanePosition:
EMPTY_POSITION = LanePosition(position=None, applied={}, key_columns=())
-# Columns that never take part in a content comparison: the position and operation are the key,
-# the SCD2 pair is stamped after replay, and the TOAST marker names columns to leave out per row.
-NON_CONTENT_COLUMNS = frozenset({SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN})
+def non_content_columns(table_columns: Iterable[str]) -> frozenset[str]:
+ """Columns that never take part in a content comparison, beyond every `_ph_` column.
-def content_columns(names: list[str]) -> list[str]:
- return [n for n in names if not n.startswith("_ph_") and n not in NON_CONTENT_COLUMNS]
+ The position and operation are the key, the SCD2 pair is stamped after replay, and the TOAST marker
+ names columns to leave out per row. A legacy history table stamps `valid_from` / `valid_to`. A new
+ one stamps `_ph_` names, so its `valid_from` and `valid_to` are the source's own and count.
+ """
+ if history_naming_of(table_columns) == HISTORY_NAMING:
+ return frozenset()
+ return LEGACY_HISTORY_NAMING.validity_columns
+
+
+def content_columns(names: list[str], non_content: frozenset[str]) -> list[str]:
+ return [n for n in names if not n.startswith("_ph_") and n not in non_content]
def _has_position_column(delta_table: deltalake.DeltaTable) -> bool:
@@ -153,11 +163,13 @@ async def read_lane_position(
present = {field.name for field in delta_table.schema().fields}
columns = [name for name in key_columns if name in present]
candidates = _files_at_position(add_actions, highest)
+ non_content = non_content_columns(present)
return LanePosition(
position=highest,
applied={},
key_columns=tuple(columns),
- load_applied=partial(_load_applied, delta_table, add_actions, highest, candidates, columns),
+ non_content_columns=non_content,
+ load_applied=partial(_load_applied, delta_table, add_actions, highest, candidates, columns, non_content),
)
@@ -167,6 +179,7 @@ def _load_applied(
highest: int,
candidates: dict[str, int],
columns: list[str],
+ non_content: frozenset[str],
) -> LanePosition:
"""The rows the table holds at `highest`, read from the candidate files a batch at a time.
@@ -210,13 +223,15 @@ def _load_applied(
applied={key: [placeholder] * count for key, count in counts.items()},
key_columns=tuple(columns),
content_matched=False,
+ non_content_columns=non_content,
)
at_position = pa.Table.from_batches(batches, schema=schema)
return LanePosition(
position=highest,
- applied=_group_by_identity(at_position, columns),
+ applied=_group_by_identity(at_position, columns, non_content),
key_columns=tuple(columns),
- content_schema=at_position.select(content_columns(at_position.column_names)).schema,
+ content_schema=at_position.select(content_columns(at_position.column_names, non_content)).schema,
+ non_content_columns=non_content,
)
@@ -306,11 +321,13 @@ def _all_null(add_actions: pa.Table) -> set[str]:
}
-def _group_by_identity(rows: pa.Table, key_columns: list[str]) -> dict[tuple[Any, ...], list[dict[str, Any]]]:
+def _group_by_identity(
+ rows: pa.Table, key_columns: list[str], non_content: frozenset[str]
+) -> dict[tuple[Any, ...], list[dict[str, Any]]]:
if not key_columns or not rows.num_rows:
return {}
keys = _identities(rows, key_columns)
- contents = rows.select(content_columns(rows.column_names)).to_pylist()
+ contents = rows.select(content_columns(rows.column_names, non_content)).to_pylist()
grouped: defaultdict[tuple[Any, ...], list[dict[str, Any]]] = defaultdict(list)
for key, content in zip(keys, contents):
grouped[key].append(content)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py
index 5a11203067b5..fa29f90ed3b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/load_resolution.py
@@ -13,10 +13,11 @@ import pyarrow as pa
from posthog.dataclasses import frozen
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
- _CDC_METADATA_COLUMNS,
CDC_OP_COLUMN,
CDC_SEQ_COLUMN,
CDC_SEQ_PROVENANCE,
+ HistoryNaming,
+ metadata_columns,
)
# Verification is a diagnostic; cap it so a delete-heavy batch can't dominate the write path.
@@ -43,9 +44,10 @@ class DeleteEnrichmentReport:
return self.rows_with_nulled_columns == 0
-def _data_columns(table: pa.Table, primary_keys: list[str]) -> list[str]:
+def _data_columns(table: pa.Table, primary_keys: list[str], history: HistoryNaming | None) -> list[str]:
pk_set = set(primary_keys)
- return [c for c in table.column_names if c not in _CDC_METADATA_COLUMNS and c not in pk_set]
+ metadata = metadata_columns(history)
+ return [c for c in table.column_names if c not in metadata and c not in pk_set]
def _pk_tuples(table: pa.Table, primary_keys: list[str]) -> list[tuple]:
@@ -136,6 +138,7 @@ def verify_delete_enrichment(
table: pa.Table,
primary_keys: list[str],
existing_rows: pa.Table | None,
+ history: HistoryNaming | None = None,
) -> DeleteEnrichmentReport:
"""Report DELETE rows that would null a data column the target still holds.
@@ -152,7 +155,7 @@ def verify_delete_enrichment(
if not present_pks:
return empty
- data_cols = [c for c in _data_columns(table, present_pks) if c in existing_rows.column_names]
+ data_cols = [c for c in _data_columns(table, present_pks, history) if c in existing_rows.column_names]
if not data_cols:
return empty
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.py
index 91e13d302a94..13b2c1e36658 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_manager.py
@@ -34,8 +34,8 @@ from products.data_warehouse.backend.facade.api import aget_s3_client
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_OP_COLUMN,
CDC_SEQ_COLUMN,
- SCD2_VALID_TO_COLUMN,
TOAST_OMITTED_COLUMN,
+ HistoryNaming,
build_scd2_table,
companion_resource_name,
)
@@ -106,6 +106,7 @@ class CDCLane:
resource_name: str
write_mode: CDCWriteMode
+ history: HistoryNaming | None = None
def serves_buffered_lane(schema: ExternalDataSchema) -> bool:
@@ -150,23 +151,23 @@ def purge_buffer_before_handover(schema: ExternalDataSchema, logger: FilteringBo
purge_buffer_prefix(schema.team_id, str(schema.id), logger, strict=True)
-def served_lanes(schema: ExternalDataSchema) -> list[CDCLane]:
- """The tables this schema's change stream feeds.
+def served_lanes(schema: ExternalDataSchema, history: HistoryNaming | None) -> list[CDCLane]:
+ """The tables this schema's change stream feeds. `history` names the history table, when the mode writes one.
One entry per Delta table the mode writes, in the order a run writes them.
An unrecognized mode returns nothing, which reads as "not a lane the buffer serves".
"""
- return [
- CDCLane(
- resource_name=(
- companion_resource_name(schema.name)
- if mode == COMPANION_WRITE_MODE
- else consolidated_resource_name(schema)
- ),
- write_mode=mode,
- )
- for mode in _LANE_WRITE_MODES.get(schema.cdc_table_mode, ())
- ]
+ lanes: list[CDCLane] = []
+ for mode in _LANE_WRITE_MODES.get(schema.cdc_table_mode, ()):
+ if mode != COMPANION_WRITE_MODE:
+ lanes.append(CDCLane(resource_name=consolidated_resource_name(schema), write_mode=mode))
+ elif history is None:
+ raise ValueError(f"CDC schema {schema.name} writes a history table, but no history naming was given")
+ else:
+ lanes.append(
+ CDCLane(resource_name=companion_resource_name(schema.name, history), write_mode=mode, history=history)
+ )
+ return lanes
# Slack when comparing an S3 mtime against a listing timestamp from our clock, so skew between the
@@ -331,7 +332,9 @@ _PROOF_SEARCH_DEPTH = 10
_PROOF_WINDOW = dt.timedelta(days=2)
-def _history_transform(replay: ReplayFilter, key_columns: list[str]) -> Callable[[pa.Table], pa.Table]:
+def _history_transform(
+ replay: ReplayFilter, key_columns: list[str], history: HistoryNaming
+) -> Callable[[pa.Table], pa.Table]:
"""Replay first, then stamp the SCD2 validity columns onto what is left.
Derived here rather than in the loader so the staged parquet is complete on its own: a loader
@@ -349,16 +352,16 @@ def _history_transform(replay: ReplayFilter, key_columns: list[str]) -> Callable
table = replay.apply(table)
if not table.num_rows:
return table
- # No guard on the columns already being there: a source that owns `valid_from` fails
+ # No guard on the columns already being there: a source that owns a validity column fails
# loudly inside `build_scd2_table`, where a skip would let the writer close rows against
# customer data.
- return build_scd2_table(table, key_columns)
+ return build_scd2_table(table, key_columns, history)
return _apply
async def build_output_lanes(
- schema: ExternalDataSchema, job: ExternalDataJob, logger: FilteringBoundLogger
+ schema: ExternalDataSchema, job: ExternalDataJob, logger: FilteringBoundLogger, history: HistoryNaming | None
) -> tuple[list[OutputLane], int | None]:
"""Every table this run writes, and the position below which the buffer is settled.
@@ -374,7 +377,7 @@ async def build_output_lanes(
"""
lanes: list[OutputLane] = []
positions: list[int | None] = []
- for index, lane in enumerate(served_lanes(schema)):
+ for index, lane in enumerate(served_lanes(schema, history)):
delta_table = await DeltaTableRef(
lane.resource_name, job, logger, expect_missing=schema.table_id is None
).get_delta_table()
@@ -382,7 +385,7 @@ async def build_output_lanes(
keys = [normalize_column_name(name) for name in schema.primary_key_columns or []]
if delta_table is not None:
# Before the read, so this run's own write is the one that carries the statistic.
- await ensure_position_stats(delta_table, [*keys, *([SCD2_VALID_TO_COLUMN] if is_append else [])])
+ await ensure_position_stats(delta_table, [*keys, *([lane.history.valid_to] if lane.history else [])])
# Without primary keys an identity is just the operation, which matches rows the table
# has never held. Ask for none instead: the lane then replays, which duplicates history
# rather than losing it, and duplication is the direction that can be repaired.
@@ -395,7 +398,7 @@ async def build_output_lanes(
name=lane.resource_name,
cdc_write_mode=lane.write_mode,
billable=index == 0,
- transform=_history_transform(replay, keys) if is_append else replay.apply,
+ transform=_history_transform(replay, keys, lane.history) if lane.history else replay.apply,
)
)
floor = None if any(p is None for p in positions) else min(p for p in positions if p is not None)
@@ -521,6 +524,7 @@ class ReplayFilter:
# Taken from the position itself, so the batch is keyed exactly as the table was read.
self._key_columns = list(position.key_columns)
self._content_schema = position.content_schema
+ self._non_content = position.non_content_columns
self._content_matched = position.content_matched
self._load_applied = position.load_applied
self.rows_skipped = 0
@@ -541,6 +545,7 @@ class ReplayFilter:
self._load_applied = None
self._applied = {key: list(rows) for key, rows in resolved.applied.items()}
self._content_schema = resolved.content_schema
+ self._non_content = resolved.non_content_columns
self._content_matched = resolved.content_matched
def _count_skipped(self, dropped: int, reason: str) -> None:
@@ -612,7 +617,7 @@ class ReplayFilter:
still compare different: dropping it would make a wrong match likelier, and a wrong match
is a lost change where a missed one is only a copy.
"""
- names = content_columns(table.column_names)
+ names = content_columns(table.column_names, self._non_content)
if self._content_schema is None:
return table.select(names).to_pylist(), set()
columns: list[pa.ChunkedArray] = []
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py
index cecc7f5737a1..c15378ebdf82 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py
@@ -89,6 +89,7 @@ class CDCSourceMixin(_BaseSource[ConfigType]):
from products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobs import (
retire_orphaned_companions,
)
+ from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import kept_history_naming
from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import hand_reset_to_capture
from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (
CDCSourceManager,
@@ -100,7 +101,8 @@ class CDCSourceMixin(_BaseSource[ConfigType]):
served_lanes,
)
- if not served_lanes(schema):
+ history = kept_history_naming(schema)
+ if not served_lanes(schema, history):
raise ValueError(
f"CDC schema {schema.name} has cdc_table_mode {schema.cdc_table_mode!r}, which no buffer lane "
"writes. Set it to 'consolidated', 'cdc_only' or 'both'."
@@ -113,7 +115,7 @@ class CDCSourceMixin(_BaseSource[ConfigType]):
# of that attempt failing later would leave a Completed job proving a listing nothing
# drained.
clear_listing(inputs.job_id, inputs.team_id)
- first_lane = served_lanes(schema)[0]
+ first_lane = served_lanes(schema, history)[0]
return SourceResponse(
name=first_lane.resource_name,
items=lambda: iter(()),
@@ -167,7 +169,7 @@ class CDCSourceMixin(_BaseSource[ConfigType]):
# Every table this schema's changes feed, written from one read of the buffer. Each lane
# carries where its own table stops, because a failed run can leave one ahead of the other.
- lanes, deletion_floor = async_to_sync(build_output_lanes)(schema, job, inputs.logger)
+ lanes, deletion_floor = async_to_sync(build_output_lanes)(schema, job, inputs.logger, history)
manager = CDCSourceManager(
inputs,
inputs.logger,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
index 38deac34afb9..f711d760c24f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
@@ -37,13 +37,19 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
CDC_SEQ_COLUMN,
CDC_TIMESTAMP_COLUMN,
DELETED_COLUMN,
+ HISTORY_NAMING,
TOAST_OMITTED_COLUMN,
companion_resource_name,
enrich_delete_rows,
enrich_toast_omitted_rows,
+ history_naming_of,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.buffer import get_buffer_prefix
-from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import LanePosition, content_columns
+from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import (
+ LanePosition,
+ content_columns,
+ non_content_columns,
+)
from products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import resolve_batch
from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (
COMPANION_WRITE_MODE,
@@ -336,11 +342,14 @@ def lane_position(rows: Sequence[Row], key_columns: Sequence[str] | None) -> Lan
if position is None or not key_columns:
return LanePosition(position=position, applied={})
applied: defaultdict[Key, list[Row]] = defaultdict(list)
+ non_content = non_content_columns({column for row in rows for column in row})
for row in rows:
if row.get(CDC_SEQ_COLUMN) == position:
identity = tuple(row.get(column) for column in key_columns)
- applied[identity].append({column: row[column] for column in content_columns(list(row))})
- return LanePosition(position=position, applied=dict(applied), key_columns=tuple(key_columns))
+ applied[identity].append({column: row[column] for column in content_columns(list(row), non_content)})
+ return LanePosition(
+ position=position, applied=dict(applied), key_columns=tuple(key_columns), non_content_columns=non_content
+ )
class Warehouse:
@@ -374,7 +383,7 @@ class Warehouse:
current = {self._table.key_of(row): row for row in self.history}
existing = self._as_table(list(current.values()))
batch = enrich_toast_omitted_rows(batch, self._keys, existing)
- batch = enrich_delete_rows(batch, self._keys, existing)
+ batch = enrich_delete_rows(batch, self._keys, existing, history_naming_of(batch.column_names))
self.history.extend(_without_toast_marker(batch).to_pylist())
def reseed(self, rows: dict[Key, Row]) -> None:
@@ -531,6 +540,11 @@ class ConformanceKit:
self._engine.installed(),
patch(f"{_CDC}.activities.get_cdc_adapter", return_value=self._engine.adapter),
patch(f"{_CDC}.activities.source_type_supports_cdc", return_value=True),
+ # The kit registers no tables, so every history table is a new one.
+ patch(
+ f"{_CDC}.activities.SourceHistoryNamings",
+ return_value=MagicMock(naming_unless_taken=MagicMock(return_value=HISTORY_NAMING)),
+ ),
patch(f"{_CDC}.activities.ExternalDataSource", source_model),
patch.object(CDCExtractActivity, "_get_cdc_schemas", cdc_schemas),
patch(f"{_CDC}.activities.update_sync_type_config_keys", merge_config),
@@ -587,12 +601,12 @@ class ConformanceKit:
schema, warehouse = self.schema(table), self._warehouses[table.name]
self._lane_rows = {
consolidated_resource_name(schema): lambda: list(warehouse.merged.values()),
- companion_resource_name(schema.name): lambda: warehouse.history,
+ companion_resource_name(schema.name, HISTORY_NAMING): lambda: warehouse.history,
}
inputs = SimpleNamespace(team_id=_TEAM_ID, schema_id=str(schema.id), job_id="job")
async def drain() -> None:
- lanes, floor = await build_output_lanes(schema, MagicMock(), self._log)
+ lanes, floor = await build_output_lanes(schema, MagicMock(), self._log, HISTORY_NAMING)
manager = CDCSourceManager(
cast(SourceInputs, inputs), self._log, deletion_floor=floor, proof=self._proofs.get(table.name)
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
index 1a11ed1f6cab..e01cd0aa7efc 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
@@ -15,10 +15,11 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
CDC_TIMESTAMP_COLUMN,
DELETED_AT_COLUMN,
DELETED_COLUMN,
- SCD2_VALID_FROM_COLUMN,
- SCD2_VALID_TO_COLUMN,
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
TOAST_OMITTED_COLUMN,
ChangeEventBatcher,
+ HistoryNaming,
build_scd2_table,
enrich_delete_rows,
enrich_toast_omitted_rows,
@@ -365,15 +366,15 @@ class TestBuildScd2Table:
for i, (op, cols, ts) in enumerate(ops_cols_ts, start=1)
]
table = self._make_raw_table(events)
- result = build_scd2_table(table, pk_columns)
- assert result.column(SCD2_VALID_TO_COLUMN).to_pylist() == expected_valid_to
- assert result.column(SCD2_VALID_FROM_COLUMN).to_pylist() == expected_valid_from
+ result = build_scd2_table(table, pk_columns, HISTORY_NAMING)
+ assert result.column(HISTORY_NAMING.valid_to).to_pylist() == expected_valid_to
+ assert result.column(HISTORY_NAMING.valid_from).to_pylist() == expected_valid_from
def test_valid_from_equals_cdc_timestamp(self):
ts = datetime(2026, 3, 15, 10, 30, 0, tzinfo=UTC)
table = self._make_raw_table([_make_event(timestamp=ts)])
- result = build_scd2_table(table, ["id"])
- assert result.column(SCD2_VALID_FROM_COLUMN)[0].as_py() == ts
+ result = build_scd2_table(table, ["id"], HISTORY_NAMING)
+ assert result.column(HISTORY_NAMING.valid_from)[0].as_py() == ts
assert result.column(CDC_TIMESTAMP_COLUMN)[0].as_py() == ts
@@ -444,6 +445,20 @@ class TestEnrichDeleteRows:
assert result.column(DELETED_AT_COLUMN)[1].as_py() == ts_delete
assert result.column(DELETED_COLUMN)[1].as_py() is True
+ def test_only_the_stamped_validity_columns_are_metadata(self):
+ # A new history table frees `valid_from` for the source, so a delete must carry the source's
+ # last value for it while keeping its own stamp.
+ ts_insert = datetime(2025, 1, 1, tzinfo=UTC)
+ ts_delete = datetime(2025, 6, 1, tzinfo=UTC)
+ events = [
+ _make_event(op="I", columns={"id": 1, "valid_from": "2020-01-01"}, timestamp=ts_insert),
+ _make_event(op="D", columns={"id": 1}, timestamp=ts_delete),
+ ]
+ table = build_scd2_table(self._make_raw_table(events), ["id"], HISTORY_NAMING)
+ result = enrich_delete_rows(table, ["id"], history=HISTORY_NAMING)
+ assert result.column("valid_from")[1].as_py() == "2020-01-01"
+ assert result.column(HISTORY_NAMING.valid_from)[1].as_py() == ts_delete
+
def test_empty_pk_columns_returns_unchanged(self):
table = self._make_raw_table([_make_event(op="D", columns={"id": 1})])
result = enrich_delete_rows(table, [])
@@ -648,7 +663,7 @@ class TestSeqColumn:
assert has_engine_seq(table)
assert has_engine_seq(toasted)
assert has_engine_seq(enriched)
- assert has_engine_seq(build_scd2_table(enriched, ["id"]))
+ assert has_engine_seq(build_scd2_table(enriched, ["id"], HISTORY_NAMING))
def test_engine_seq_is_stamped_and_survives_a_parquet_round_trip(self):
# The load side tells our column from a same-named source column by this stamp, and it
@@ -705,29 +720,52 @@ class TestScd2TimestampType:
}
)
- result = build_scd2_table(table, ["id"])
+ result = build_scd2_table(table, ["id"], HISTORY_NAMING)
- assert result.schema.field(SCD2_VALID_FROM_COLUMN).type == ts
- assert result.schema.field(SCD2_VALID_TO_COLUMN).type == ts
- assert result.column(SCD2_VALID_TO_COLUMN).to_pylist()[0] is not None
+ assert result.schema.field(HISTORY_NAMING.valid_from).type == ts
+ assert result.schema.field(HISTORY_NAMING.valid_to).type == ts
+ assert result.column(HISTORY_NAMING.valid_to).to_pylist()[0] is not None
class TestReservedColumnCollisions:
@parameterized.expand(
[
- ("exact_name", ["id", "valid_to"], True, None, None, ["valid_to"]),
- ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], True, None, None, ["VALID_TO", "validFrom"]),
- ("capture_name_the_load_renames", ["id", "_PH_CDC_SEQ"], False, None, None, ["_PH_CDC_SEQ"]),
- ("validity_name_without_history", ["id", "valid_from"], False, None, None, []),
- ("left_out_of_enabled_columns", ["id", "validFrom"], True, ["id"], None, []),
- ("kept_as_the_incremental_field", ["id", "valid_from"], True, [], "valid_from", ["valid_from"]),
+ ("legacy_validity_name", ["id", "valid_to"], LEGACY_HISTORY_NAMING, None, None, ["valid_to"]),
+ (
+ "legacy_name_the_load_renames",
+ ["id", "validFrom", "VALID_TO"],
+ LEGACY_HISTORY_NAMING,
+ None,
+ None,
+ ["VALID_TO", "validFrom"],
+ ),
+ ("legacy_validity_name_on_a_new_table", ["id", "valid_from"], HISTORY_NAMING, None, None, []),
+ (
+ "new_validity_name_on_a_legacy_table",
+ ["id", "_PH_VALID_FROM"],
+ LEGACY_HISTORY_NAMING,
+ None,
+ None,
+ ["_PH_VALID_FROM"],
+ ),
+ ("capture_name_the_load_renames", ["id", "_PH_CDC_SEQ"], None, None, None, ["_PH_CDC_SEQ"]),
+ ("validity_name_without_history", ["id", "valid_from"], None, None, None, []),
+ ("left_out_of_enabled_columns", ["id", "validFrom"], LEGACY_HISTORY_NAMING, ["id"], None, []),
+ (
+ "kept_as_the_incremental_field",
+ ["id", "valid_from"],
+ LEGACY_HISTORY_NAMING,
+ [],
+ "valid_from",
+ ["valid_from"],
+ ),
]
)
def test_names_the_source_columns_that_collide(
self,
_name: str,
column_names: list[str],
- history: bool,
+ history: HistoryNaming | None,
enabled_columns: list[str] | None,
incremental_field: str | None,
expected: list[str],
@@ -735,7 +773,7 @@ class TestReservedColumnCollisions:
assert (
reserved_column_collisions(
column_names,
- reserved_column_names(history=history),
+ reserved_column_names(history),
enabled_columns=enabled_columns,
primary_keys=["id"],
incremental_field=incremental_field,
@@ -760,8 +798,16 @@ class TestIsHistoryTableMode:
class TestScd2ReservedColumns:
- @parameterized.expand([(SCD2_VALID_FROM_COLUMN,), (SCD2_VALID_TO_COLUMN,)])
- def test_a_source_column_named_like_the_stamp_fails_before_the_writer(self, taken):
+ @parameterized.expand(
+ [
+ ("legacy_valid_from", LEGACY_HISTORY_NAMING, LEGACY_HISTORY_NAMING.valid_from),
+ ("legacy_valid_to", LEGACY_HISTORY_NAMING, LEGACY_HISTORY_NAMING.valid_to),
+ ("new_valid_from", HISTORY_NAMING, HISTORY_NAMING.valid_from),
+ ]
+ )
+ def test_a_source_column_named_like_the_stamp_fails_before_the_writer(
+ self, _name: str, history: HistoryNaming, taken: str
+ ) -> None:
# Delta refuses the duplicate name at write time, but by then a batch is half built and
# the error names a qualified field, not the customer's column. A silent skip here is
# worse still: the writer would close rows against the customer's own values.
@@ -776,4 +822,4 @@ class TestScd2ReservedColumns:
)
with pytest.raises(CDCReservedColumnError, match=taken):
- build_scd2_table(table, ["id"])
+ build_scd2_table(table, ["id"], history)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.py
index 27b0d710a13e..aa6360c62ec7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_buffered_dispatch.py
@@ -4,6 +4,7 @@ import pytest
from unittest.mock import ANY, AsyncMock, MagicMock, call, patch
from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import LEGACY_HISTORY_NAMING
from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import LanePosition
from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import ListingProof
from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
@@ -15,6 +16,7 @@ _MANAGER = "products.warehouse_sources.backend.temporal.data_imports.cdc.source_
_COMPANIONS = "products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobs"
_SOURCE = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source"
_SNAPSHOT_LANE = "products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane"
+_HISTORY_TABLE = "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table"
def _schema(**overrides) -> MagicMock:
@@ -89,6 +91,11 @@ def _dispatch(
patch(f"{_MANAGER}.completed_listing_proof", AsyncMock(return_value=proof)),
patch(f"{_MANAGER}.buffer_expired_unread", expired or AsyncMock(return_value=False)),
patch(f"{_SNAPSHOT_LANE}.hand_reset_to_capture", hand_reset or MagicMock()),
+ # The schemas are mocks, so the registered naming cannot be read. Every history table here is legacy.
+ patch(
+ f"{_HISTORY_TABLE}.kept_history_naming",
+ side_effect=lambda schema: LEGACY_HISTORY_NAMING if schema.cdc_table_mode in ("cdc_only", "both") else None,
+ ),
patch.object(PostgresSource, "make_ssh_tunnel_func", return_value=MagicMock()),
):
objects.select_related.return_value.get.return_value = schema
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
index 10d20de3173c..ae0b7e550cd4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
@@ -34,7 +34,10 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
CDC_OP_COLUMN,
CDC_SEQ_COLUMN,
CDC_SEQ_PROVENANCE,
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
ChangeEventBatcher,
+ HistoryNaming,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import (
CDCErrorCategory,
@@ -2342,22 +2345,28 @@ class TestBufferedIngressCapture:
@parameterized.expand(
[
- ("position_column", CDC_SEQ_COLUMN, "consolidated", True),
- ("position_column_the_load_renames", "_PH_CDC_SEQ", "consolidated", True),
- ("validity_column_with_a_history_table", "validFrom", "both", True),
- ("validity_column_without_a_history_table", "valid_from", "consolidated", False),
+ ("position_column", CDC_SEQ_COLUMN, None, True),
+ ("position_column_the_load_renames", "_PH_CDC_SEQ", None, True),
+ ("validity_column_on_a_legacy_history_table", "validFrom", LEGACY_HISTORY_NAMING, True),
+ ("legacy_validity_column_on_a_new_history_table", "valid_from", HISTORY_NAMING, False),
+ ("validity_column_without_a_history_table", "valid_from", None, False),
]
)
def test_a_source_column_named_like_a_written_column_fails_the_buffered_run(
- self, _name: str, column: str, cdc_table_mode: str, refused: bool
+ self, _name: str, column: str, history: HistoryNaming | None, refused: bool
) -> None:
# Written anyway, the file's name, ordering, and retry cleanup could derive from customer
# data, and cleanup can then delete unconsumed files.
source = _make_source()
+ cdc_table_mode = "consolidated" if history is None else "both"
schema = _make_schema("users", cdc_mode="streaming", cdc_table_mode=cdc_table_mode, source=source)
events = [_make_event(op="I", position="0/100", columns={"id": 1, column: 42})]
- with _capture_harness(source, [schema], events) as capture:
+ with (
+ patch(f"{_ACTIVITIES}.SourceHistoryNamings") as history_namings,
+ _capture_harness(source, [schema], events) as capture,
+ ):
+ history_namings.return_value.naming_unless_taken.return_value = history
if refused:
with pytest.raises(NonRetryableException, match="change data capture"):
capture.extract()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py
new file mode 100644
index 000000000000..c24b8ee21b43
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py
@@ -0,0 +1,192 @@
+import uuid
+
+import pytest
+from posthog.test.base import BaseTest
+from unittest.mock import patch
+
+from parameterized import parameterized
+
+from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
+ HistoryNaming,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (
+ HistoryTableNameTakenError,
+ history_naming,
+ history_tables,
+ is_another_schemas_history_table,
+ new_history_names_taken,
+ pin_history_naming,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name
+
+_PH_HISTORY_NAMING_FLAG = (
+ "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"
+)
+
+
+@patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)
+class TestHistoryNaming(BaseTest):
+ def setUp(self) -> None:
+ super().setUp()
+ self.source = ExternalDataSource.objects.create(
+ team=self.team, source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), source_type="Postgres"
+ )
+ self.schema = ExternalDataSchema.objects.create(
+ name="orders",
+ team=self.team,
+ source=self.source,
+ sync_type=ExternalDataSchema.SyncType.CDC,
+ sync_type_config={"cdc_table_mode": "both"},
+ )
+
+ def _table(self, storage_name: str, options: dict | None = None) -> DataWarehouseTable:
+ return DataWarehouseTable.objects.create(
+ team=self.team,
+ name=build_table_name(self.source, storage_name),
+ format="DeltaS3Wrapper",
+ url_pattern=f"s3://bucket/{storage_name}",
+ external_data_source=self.source,
+ options=options or {},
+ )
+
+ @parameterized.expand(
+ [
+ ("no_history_table_yet", None, None, HISTORY_NAMING),
+ ("legacy_table_without_the_marker", "orders_cdc", {}, LEGACY_HISTORY_NAMING),
+ ("legacy_table_with_the_marker", "orders_cdc", {"history_table": True}, LEGACY_HISTORY_NAMING),
+ (
+ "stamped_new_table",
+ "orders_history",
+ {"history_table": True, "history_naming": "ph_validity"},
+ HISTORY_NAMING,
+ ),
+ ]
+ )
+ def test_reads_the_naming_from_the_registered_table(
+ self, _name: str, storage_name: str | None, options: dict | None, expected: object
+ ) -> None:
+ if storage_name is not None:
+ self._table(storage_name, options)
+
+ assert history_naming(self.schema) == expected
+
+ def test_a_new_history_table_keeps_the_legacy_naming_until_the_flag_is_on(self) -> None:
+ with patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: False):
+ assert history_naming(self.schema) == LEGACY_HISTORY_NAMING
+
+ @parameterized.expand(
+ [("pinned_new_flag_off", HISTORY_NAMING, False), ("pinned_legacy_flag_on", LEGACY_HISTORY_NAMING, True)]
+ )
+ def test_a_pinned_naming_wins_over_the_flag(self, _name: str, pinned: HistoryNaming, flag_on: bool) -> None:
+ pin_history_naming(self.schema.sync_type_config, pinned)
+ self.schema.save()
+
+ with patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: flag_on):
+ assert history_naming(self.schema) == pinned
+
+ def test_a_stamped_new_table_wins_over_a_legacy_named_table(self) -> None:
+ # An unlinked `orders_cdc` can be a source table that has not synced yet. Taking the legacy naming
+ # for it would stop writing the stamped history table.
+ self._table("orders_history", {"history_table": True, "history_naming": "ph_validity"})
+ self._table("orders_cdc")
+
+ assert history_naming(self.schema) == HISTORY_NAMING
+
+ def test_a_sibling_whose_catalog_name_differs_does_not_block_history(self) -> None:
+ # `ordersHistory` normalizes to the same folder leaf, but each schema's folder carries its id.
+ ExternalDataSchema.objects.create(name="ordersHistory", team=self.team, source=self.source, should_sync=True)
+
+ assert history_naming(self.schema) == HISTORY_NAMING
+
+ @parameterized.expand(
+ [("new_naming", HISTORY_NAMING, "orders_history"), ("legacy_naming", LEGACY_HISTORY_NAMING, "orders_cdc")]
+ )
+ def test_new_history_names_taken_compares_the_payload_of_a_new_source(
+ self, _name: str, naming: HistoryNaming, sibling: str
+ ) -> None:
+ taken = new_history_names_taken(self.source, naming, ["orders", "items"], ["orders", sibling, "items"])
+
+ assert taken == ["orders"]
+
+ def test_a_sibling_source_table_named_like_the_legacy_table_is_not_its_history(self) -> None:
+ sibling = ExternalDataSchema.objects.create(name="orders_cdc", team=self.team, source=self.source)
+ sibling.table = self._table("orders_cdc")
+ sibling.save()
+
+ assert history_naming(self.schema) == HISTORY_NAMING
+
+ def test_refuses_a_name_another_table_holds(self) -> None:
+ self._table("orders_history")
+
+ with pytest.raises(HistoryTableNameTakenError):
+ history_naming(self.schema)
+
+ @parameterized.expand([("syncing", True, True), ("discovered_only", False, False)])
+ def test_refuses_a_name_a_sibling_table_syncs_under(self, _name: str, should_sync: bool, refused: bool) -> None:
+ ExternalDataSchema.objects.create(
+ name="orders_history", team=self.team, source=self.source, should_sync=should_sync
+ )
+
+ if refused:
+ with pytest.raises(HistoryTableNameTakenError):
+ history_naming(self.schema)
+ else:
+ assert history_naming(self.schema) == HISTORY_NAMING
+
+ @parameterized.expand(
+ [
+ ("new_table", "orders_history", {"history_table": True, "history_naming": "ph_validity"}),
+ # A history table that stopped syncing before the marker existed never got it.
+ ("unmarked_legacy_table", "orders_cdc", {}),
+ ]
+ )
+ def test_a_schema_cannot_take_the_name_of_another_schemas_history_table(
+ self, _name: str, storage_name: str, options: dict
+ ) -> None:
+ self._table(storage_name, options)
+ sibling = ExternalDataSchema.objects.create(name=storage_name, team=self.team, source=self.source)
+
+ assert is_another_schemas_history_table(sibling)
+ assert not is_another_schemas_history_table(self.schema)
+
+ # A schema renamed by the multi-schema migration keeps its pinned catalog name, not the name of its row.
+ @parameterized.expand([("unpinned", None, True), ("pinned_to_old_name", "orders_history", False)])
+ def test_the_reverse_guard_compares_the_catalog_name(
+ self, _name: str, s3_folder_name: str | None, refused: bool
+ ) -> None:
+ # `orders` would own `orders_history` itself.
+ self.schema.sync_type_config = {"cdc_table_mode": "consolidated"}
+ self.schema.save()
+ ExternalDataSchema.objects.create(
+ name="public.orders",
+ team=self.team,
+ source=self.source,
+ sync_type=ExternalDataSchema.SyncType.CDC,
+ sync_type_config={"cdc_table_mode": "both"},
+ )
+ sibling = ExternalDataSchema.objects.create(
+ name="public.orders_history", team=self.team, source=self.source, s3_folder_name=s3_folder_name
+ )
+
+ assert is_another_schemas_history_table(sibling) == refused
+
+ def test_history_tables_finds_marked_and_unmarked_legacy_tables(self) -> None:
+ legacy = self._table("orders_cdc")
+ marked = self._table("other_history", {"history_table": True})
+ self._table("orders")
+ self._table("customer_cdc")
+ # A discovered `orders_cdc` source table that does not sync owns no table, so the legacy table above
+ # is still the history of `orders`.
+ ExternalDataSchema.objects.create(name="orders_cdc", team=self.team, source=self.source)
+ # `items_cdc` syncs into its own table, which is not the history of `items`.
+ ExternalDataSchema.objects.create(name="items", team=self.team, source=self.source)
+ ExternalDataSchema.objects.create(
+ name="items_cdc", team=self.team, source=self.source, table=self._table("items_cdc")
+ )
+
+ assert set(history_tables(self.source)) == {legacy, marked}
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.py
index 3bc218144156..50d6e1976a2a 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_lane_position.py
@@ -6,7 +6,12 @@ import deltalake
import pyarrow.dataset as pa_ds
from products.warehouse_sources.backend.temporal.data_imports.cdc import lane_position
-from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import CDC_OP_COLUMN, CDC_SEQ_COLUMN
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ CDC_OP_COLUMN,
+ CDC_SEQ_COLUMN,
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
+)
from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import (
STATS_COLUMNS_PROPERTY,
ensure_position_stats,
@@ -95,6 +100,27 @@ class TestReadLanePosition:
assert position.content_schema is not None
assert position.content_schema.names == ["id", *(f"c{i}" for i in range(_WIDE))]
+ @pytest.mark.parametrize(
+ ("history", "compared"),
+ [(HISTORY_NAMING, ["id", "valid_from"]), (LEGACY_HISTORY_NAMING, ["id"])],
+ ids=["new_table", "legacy_table"],
+ )
+ async def test_only_the_stamped_validity_columns_stay_out_of_the_comparison(self, tmp_path, history, compared):
+ # A new table frees `valid_from` for the source. Left out, two changes to one key that differ
+ # only there would compare equal, and the second would be dropped as already written.
+ rows = _rows([30], wide=False, ids=[1])
+ if history == HISTORY_NAMING:
+ rows = rows.append_column("valid_from", pa.array(["2020-01-01"]))
+ stamp = pa.array([None], pa.timestamp("us"))
+ rows = rows.append_column(history.valid_from, stamp).append_column(history.valid_to, stamp)
+ path = str(tmp_path / "t")
+ deltalake.write_deltalake(path, rows, configuration={STATS_COLUMNS_PROPERTY: CDC_SEQ_COLUMN})
+
+ position = await _resolved(deltalake.DeltaTable(path))
+
+ assert position.content_schema is not None
+ assert position.content_schema.names == compared
+
async def test_rows_at_the_position_are_gathered_across_the_files_holding_them(self, tmp_path):
path = tmp_path / "t"
_write(path, [10, 30], ids=[1, 2])
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.py
index f308b9fb2943..39dd4b249f88 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_source_manager.py
@@ -14,8 +14,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
CDC_SEQ_COLUMN,
CDC_SEQ_PROVENANCE,
CDC_TIMESTAMP_COLUMN,
- SCD2_VALID_FROM_COLUMN,
- SCD2_VALID_TO_COLUMN,
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
TOAST_OMITTED_COLUMN,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.buffer import build_buffer_file_name
@@ -306,30 +306,41 @@ class TestServedLanes:
@parameterized.expand(
[
("consolidated", "consolidated", [CDCLane(resource_name="users", write_mode="incremental_merge")]),
- ("cdc_only", "cdc_only", [CDCLane(resource_name="users_cdc", write_mode="scd2_append")]),
+ (
+ "cdc_only",
+ "cdc_only",
+ [CDCLane(resource_name="users_history", write_mode="scd2_append", history=HISTORY_NAMING)],
+ ),
(
"both",
"both",
[
CDCLane(resource_name="users", write_mode="incremental_merge"),
- CDCLane(resource_name="users_cdc", write_mode="scd2_append"),
+ CDCLane(resource_name="users_history", write_mode="scd2_append", history=HISTORY_NAMING),
],
),
]
)
def test_a_table_mode_names_the_tables_its_changes_feed(self, _name, table_mode, expected):
- assert served_lanes(_schema(cdc_table_mode=table_mode)) == expected
+ assert served_lanes(_schema(cdc_table_mode=table_mode), HISTORY_NAMING) == expected
def test_an_unrecognized_table_mode_feeds_nothing(self):
# Fail closed: a mode this module cannot write must not reach the buffer at all.
- assert served_lanes(_schema(cdc_table_mode="something_new")) == []
+ assert served_lanes(_schema(cdc_table_mode="something_new"), HISTORY_NAMING) == []
- def test_the_companion_is_named_off_the_schema_not_the_folder(self):
+ @parameterized.expand(
+ [
+ ("new_table", HISTORY_NAMING, "public.users_history"),
+ ("legacy_table", LEGACY_HISTORY_NAMING, "public.users_cdc"),
+ ]
+ )
+ def test_the_companion_is_named_off_the_schema_not_the_folder(self, _name, history, expected):
# The consolidated lane follows the snapshot's resolved folder, which diverges from `name`
# for a row renamed bare to qualified. Following it here would append history into a table
- # no query reads — the companion is keyed on `name`, like its snapshot seed.
+ # no query reads — the companion is keyed on `name`, like its snapshot seed. A legacy history
+ # table keeps writing under its own name.
schema = _schema(name="public.users", resolved_s3_folder_name="users", cdc_table_mode="both")
- assert [lane.resource_name for lane in served_lanes(schema)] == ["users", "public.users_cdc"]
+ assert [lane.resource_name for lane in served_lanes(schema, history)] == ["users", expected]
class TestSnapshotCapture:
@@ -816,15 +827,15 @@ class TestBuildOutputLanes:
AsyncMock(side_effect=positions),
),
):
- return await build_output_lanes(schema, MagicMock(), AsyncMock())
+ return await build_output_lanes(schema, MagicMock(), AsyncMock(), HISTORY_NAMING)
async def test_both_writes_two_tables_in_one_run(self):
lanes, _ = await self._build(_schema(cdc_table_mode="both"), [_NO_POSITION, _NO_POSITION])
- assert [lane.name for lane in lanes] == ["users", "users_cdc"]
+ assert [lane.name for lane in lanes] == ["users", "users_history"]
assert [lane.cdc_write_mode for lane in lanes] == [CONSOLIDATED_WRITE_MODE, COMPANION_WRITE_MODE]
- @parameterized.expand([("consolidated", "consolidated", "users"), ("cdc_only", "cdc_only", "users_cdc")])
+ @parameterized.expand([("consolidated", "consolidated", "users"), ("cdc_only", "cdc_only", "users_history")])
async def test_a_single_table_mode_bills_its_only_lane(self, _name, table_mode, expected):
lanes, _ = await self._build(_schema(cdc_table_mode=table_mode), [_NO_POSITION])
@@ -833,7 +844,7 @@ class TestBuildOutputLanes:
async def test_both_bills_the_consolidated_lane_only(self):
lanes, _ = await self._build(_schema(cdc_table_mode="both"), [_NO_POSITION, _NO_POSITION])
- assert [(lane.name, lane.billable) for lane in lanes] == [("users", True), ("users_cdc", False)]
+ assert [(lane.name, lane.billable) for lane in lanes] == [("users", True), ("users_history", False)]
async def test_only_the_append_lane_is_asked_for_identity(self):
# Asserted on the call, not on hand-fed positions: if both lanes asked for identity the
@@ -855,14 +866,14 @@ class TestBuildOutputLanes:
"products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager.read_lane_position", reads
),
):
- await build_output_lanes(_schema(cdc_table_mode="both"), MagicMock(), AsyncMock())
+ await build_output_lanes(_schema(cdc_table_mode="both"), MagicMock(), AsyncMock(), HISTORY_NAMING)
merge_keys, append_keys = (call.kwargs["key_columns"] for call in reads.call_args_list)
assert merge_keys is None
assert append_keys == ["id", CDC_OP_COLUMN]
# The history table's own merge predicates on valid_to, so it keeps its pruning too.
- assert SCD2_VALID_TO_COLUMN in stats.call_args_list[1].args[1]
- assert SCD2_VALID_TO_COLUMN not in stats.call_args_list[0].args[1]
+ assert HISTORY_NAMING.valid_to in stats.call_args_list[1].args[1]
+ assert HISTORY_NAMING.valid_to not in stats.call_args_list[0].args[1]
async def test_a_lane_with_no_primary_keys_asks_for_no_identity(self):
# Otherwise the identity is the operation alone, which matches rows the table never held.
@@ -882,7 +893,7 @@ class TestBuildOutputLanes:
),
):
await build_output_lanes(
- _schema(cdc_table_mode="cdc_only", primary_key_columns=[]), MagicMock(), AsyncMock()
+ _schema(cdc_table_mode="cdc_only", primary_key_columns=[]), MagicMock(), AsyncMock(), HISTORY_NAMING
)
assert reads.call_args.kwargs["key_columns"] is None
@@ -907,9 +918,9 @@ class TestBuildOutputLanes:
assert history.column("id").to_pylist() == [4]
# Stamped by the lane, not the loader: a loader on the previous release has no SCD2 step
# and would append these rows with no validity at all.
- assert SCD2_VALID_FROM_COLUMN in history.column_names
- assert SCD2_VALID_TO_COLUMN in history.column_names
- assert SCD2_VALID_FROM_COLUMN not in merged.column_names
+ assert HISTORY_NAMING.valid_from in history.column_names
+ assert HISTORY_NAMING.valid_to in history.column_names
+ assert HISTORY_NAMING.valid_from not in merged.column_names
async def test_the_floor_is_the_lowest_position_any_table_holds(self):
_, floor = await self._build(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py
index e58e64c17435..f7d96399faa2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/common/load.py
@@ -183,10 +183,9 @@ async def _seed_cdc_companion_from_snapshot(
CDC_TIMESTAMP_COLUMN,
DELETED_AT_COLUMN,
DELETED_COLUMN,
- SCD2_VALID_FROM_COLUMN,
- SCD2_VALID_TO_COLUMN,
companion_resource_name as build_companion_resource_name,
)
+ from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import history_naming
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.writer import DeltaWriter
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.hogql_schema import HogQLSchema
@@ -194,6 +193,7 @@ async def _seed_cdc_companion_from_snapshot(
snapshot_dt = await snapshot_delta_table_ref.get_delta_table()
if snapshot_dt is None:
return
+ history = await database_sync_to_async_pool(history_naming)(schema)
dataset = await asyncio.to_thread(snapshot_dt.to_pyarrow_dataset)
@@ -203,12 +203,11 @@ async def _seed_cdc_companion_from_snapshot(
CDC_TIMESTAMP_COLUMN,
DELETED_COLUMN,
DELETED_AT_COLUMN,
- SCD2_VALID_FROM_COLUMN,
- SCD2_VALID_TO_COLUMN,
+ *history.validity_columns,
}
read_columns = [c for c in dataset.schema.names if c not in cdc_meta_cols]
- companion_resource_name = build_companion_resource_name(schema.name)
+ companion_resource_name = build_companion_resource_name(schema.name, history)
companion_ref = DeltaTableRef(
resource_name=companion_resource_name,
job=job,
@@ -254,8 +253,8 @@ async def _seed_cdc_companion_from_snapshot(
.append_column(pa.field(CDC_TIMESTAMP_COLUMN, ts_type), pa.array([epoch_us] * n, type=ts_type))
.append_column(pa.field(DELETED_COLUMN, pa.bool_()), pa.array([False] * n, type=pa.bool_()))
.append_column(pa.field(DELETED_AT_COLUMN, ts_type), pa.array([None] * n, type=ts_type))
- .append_column(pa.field(SCD2_VALID_FROM_COLUMN, ts_type), pa.array([epoch_us] * n, type=ts_type))
- .append_column(pa.field(SCD2_VALID_TO_COLUMN, ts_type), pa.array([None] * n, type=ts_type))
+ .append_column(pa.field(history.valid_from, ts_type), pa.array([epoch_us] * n, type=ts_type))
+ .append_column(pa.field(history.valid_to, ts_type), pa.array([None] * n, type=ts_type))
)
# Plain append — the companion table is freshly reset so there are no existing
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
index edb2a2906a3e..b46552457221 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
@@ -34,9 +34,18 @@ from products.warehouse_sources.backend.models.external_data_schema import (
update_sync_type_config_keys,
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
-from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.models.util import hogql_type_name_for_clickhouse_type
-from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import companion_resource_name
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
+ companion_resource_name,
+ history_naming_of,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (
+ HistoryTableNameTakenError,
+ not_another_schemas_table,
+)
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.db_retry import (
retry_on_operational_error,
@@ -292,20 +301,24 @@ def _record_storage_delta(job: ExternalDataJob, storage_delta_mib: Optional[floa
def own_linked_table(schema: ExternalDataSchema, pipeline: ExternalDataSource) -> DataWarehouseTable | None:
- """The schema's linked table, unless the link is its `_cdc` companion.
+ """The schema's linked table, unless the link is its history table.
- cdc_only links the schema to its companion table. Reusing that link after a switch to a mode that
- writes the consolidated table would publish the consolidated data under the companion's record,
+ cdc_only links the schema to its history table. Reusing that link after a switch to a mode that
+ writes the consolidated table would publish the consolidated data under the history table's record,
and the consolidated table would never get a record of its own. A pinned folder can give the
- schema's own table the companion's name, and then the link is right.
+ schema's own table the legacy history name, and then the link is right.
"""
table = schema.table
if table is None:
return None
names = resolve_table_and_folder_names(schema.name, schema.resolved_s3_folder_name)
- table_name = build_table_name(pipeline, names.table_storage_name)
- companion_name = build_table_name(pipeline, companion_resource_name(schema.name))
- return None if table.name == companion_name != table_name else table
+ if table.name == build_table_name(pipeline, names.table_storage_name):
+ return table
+ if table.is_history_table:
+ return None
+ # A history table registered before the marker existed carries only its legacy name.
+ legacy_name = build_table_name(pipeline, companion_resource_name(schema.name, LEGACY_HISTORY_NAMING))
+ return None if table.name == legacy_name else table
def find_orphan_table(schema: ExternalDataSchema, source: ExternalDataSource) -> DataWarehouseTable | None:
@@ -685,12 +698,19 @@ async def register_cdc_companion_table(
# when the cluster is degraded, so holding a Postgres transaction open across them is
# what produced minutes-long "idle in transaction" connections.
# Find existing companion table (not schema.table) by name
- companion_table: DataWarehouseTable | None = DataWarehouseTable.objects.filter(
+ same_name = DataWarehouseTable.objects.filter(
team_id=team_id,
name=companion_table_name,
external_data_source_id=job.pipeline.id,
deleted=False,
- ).first()
+ )
+ companion_table: DataWarehouseTable | None = not_another_schemas_table(same_name, schema_id).first()
+ if companion_table is None and same_name.exists():
+ # Another schema's table has this name. Taking its record would point it at this history,
+ # and a second record of the same name would hide one of them.
+ raise HistoryTableNameTakenError(
+ f"{companion_table_name} is already the name of another table of this source"
+ )
previous_queryable_folder = companion_table.queryable_folder if companion_table else None
storage_delta_mib = _storage_delta_mib(companion_table, live_size_mib)
@@ -717,13 +737,17 @@ async def register_cdc_companion_table(
)
else:
logger.debug(f"Creating CDC companion table: {companion_table_name}")
- companion_table = DataWarehouseTable.objects.create(
+ companion_table = DataWarehouseTable(
external_data_source_id=job.pipeline.id,
created_via=DataWarehouseTableCreatedVia.SOURCE,
size_in_s3_mib=live_size_mib,
- options={HISTORY_TABLE_OPTION: True},
**table_params,
)
+ # The batches carry the validity columns of the naming this run chose for the table.
+ companion_table.mark_as_history_table(
+ ph_naming=history_naming_of(table_schema_dict or {}) == HISTORY_NAMING
+ )
+ companion_table.save()
_record_storage_delta(job, storage_delta_mib)
_record_query_folder_pointer(
schema_id, team_id, previous_queryable_folder, queryable_folder, run_id, logger
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
index b7112f04080e..a29ec944b343 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/processor.py
@@ -29,11 +29,10 @@ from products.warehouse_sources.backend.models.external_data_schema import Exter
from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_OP_COLUMN,
- SCD2_VALID_FROM_COLUMN,
- SCD2_VALID_TO_COLUMN,
TOAST_OMITTED_COLUMN,
enrich_delete_rows,
enrich_toast_omitted_rows,
+ history_naming_of,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import (
has_engine_seq,
@@ -260,21 +259,22 @@ def _enrich_cdc_rows(
# For SCD2 tables, keep only "current" rows (valid_to IS NULL) so we
# enrich with the most recent state rather than a historical one.
+ history = history_naming_of(pa_table.column_names) if cdc_write_mode == "scd2_append" else None
if (
- cdc_write_mode == "scd2_append"
+ history is not None
and existing_rows.num_rows > 0
- and SCD2_VALID_TO_COLUMN in existing_rows.column_names
+ and history.valid_to in existing_rows.column_names
):
- existing_rows = existing_rows.filter(pc.is_null(existing_rows.column(SCD2_VALID_TO_COLUMN)))
+ existing_rows = existing_rows.filter(pc.is_null(existing_rows.column(history.valid_to)))
# TOAST fill first so DELETE enrichment copies resolved values,
# not the nulls standing in for omitted columns.
pa_table = enrich_toast_omitted_rows(pa_table, primary_keys, existing_rows)
- pa_table = enrich_delete_rows(pa_table, primary_keys, existing_rows)
+ pa_table = enrich_delete_rows(pa_table, primary_keys, existing_rows, history)
# Only place the previous state is still in hand to compare against.
if verify_deletes and delete_key_set:
- report = verify_delete_enrichment(pa_table, present_pks, existing_rows)
+ report = verify_delete_enrichment(pa_table, present_pks, existing_rows, history)
if not report.ok:
CDC_DELETE_ENRICHMENT_VIOLATIONS_TOTAL.inc(report.rows_with_nulled_columns)
logger.warning(
@@ -1661,10 +1661,10 @@ def _process_message_reported(
timer.step("write"),
DELTA_WRITE_DURATION_SECONDS.labels(write_type="scd2_append").time(),
):
+ # The lane stamped the batch with its table's validity columns, so the batch names them.
+ history = history_naming_of(pa_table.column_names)
scd2_writer = Scd2DeltaWriter(
- delta_table_ref,
- valid_from_column=SCD2_VALID_FROM_COLUMN,
- valid_to_column=SCD2_VALID_TO_COLUMN,
+ delta_table_ref, valid_from_column=history.valid_from, valid_to_column=history.valid_to
)
delta_table = async_to_sync(scd2_writer.write)(
data=pa_table,
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
index 13f1504d0504..87e047858c38 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/load/test_processor.py
@@ -20,8 +20,10 @@ from temporalio.service import RPCError, RPCStatusCode
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
CDC_OP_COLUMN,
CDC_SEQ_COLUMN,
- SCD2_VALID_TO_COLUMN,
+ HISTORY_NAMING,
+ LEGACY_HISTORY_NAMING,
TOAST_OMITTED_COLUMN,
+ HistoryNaming,
)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.common.load import PostLoadResult
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.consts import PARTITION_KEY
@@ -1737,7 +1739,8 @@ class TestEnrichCdcRows:
assert result.column("name").to_pylist() == ["renamed", "two"]
assert TOAST_OMITTED_COLUMN not in result.column_names
- def test_scd2_fill_uses_current_row_not_history(self):
+ @parameterized.expand([("legacy_table", LEGACY_HISTORY_NAMING), ("new_table", HISTORY_NAMING)])
+ def test_scd2_fill_uses_current_row_not_history(self, _name: str, history: HistoryNaming) -> None:
with tempfile.TemporaryDirectory() as path:
ts = pa.timestamp("us", tz="UTC")
write_deltalake(
@@ -1747,13 +1750,18 @@ class TestEnrichCdcRows:
"id": pa.array([1, 1], pa.int64()),
"name": pa.array(["v1", "v2"], pa.string()),
"big": pa.array(["old-toast", "current-toast"], pa.string()),
- SCD2_VALID_TO_COLUMN: pa.array([1_000_000, None], ts),
+ history.valid_from: pa.array([0, 1_000_000], ts),
+ history.valid_to: pa.array([1_000_000, None], ts),
}
),
mode="overwrite",
)
+ # The lane stamps the batch with its table's validity columns before the loader sees it.
batch = self._batch([{"id": 1, "op": "U", "name": "v3", "omitted": ["big"]}])
+ batch = batch.append_column(history.valid_from, pa.array([2_000_000], ts)).append_column(
+ history.valid_to, pa.array([None], ts)
+ )
result = _enrich_cdc_rows(
batch,
primary_keys=["id"],
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
index 3439a609ac59..0b58bd73de30 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
@@ -23,6 +23,11 @@ from products.warehouse_sources.backend.models.external_data_schema import (
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import HISTORY_NAMING, LEGACY_HISTORY_NAMING
+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import (
+ HistoryTableNameTakenError,
+ history_naming,
+)
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import (
build_table_name,
@@ -375,6 +380,71 @@ class TestRegisterCdcCompanionTableStamp:
assert history.active == "orders_cdc__query_a"
assert history.active_job_id == str(job.id)
+ @pytest.mark.parametrize(
+ ("resource_name", "validity", "expected"),
+ [
+ pytest.param("orders_history", HISTORY_NAMING, HISTORY_NAMING, id="new_table"),
+ pytest.param("orders_cdc", LEGACY_HISTORY_NAMING, LEGACY_HISTORY_NAMING, id="legacy_table"),
+ ],
+ )
+ def test_the_next_run_reads_the_naming_the_table_was_created_with(self, team, resource_name, validity, expected):
+ # Without the stamp, the next run would take its own `_history` table for another table's.
+ source = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=team, source_type="Postgres"
+ )
+ schema = ExternalDataSchema.objects.create(name="orders", team=team, source=source)
+ job = ExternalDataJob.objects.create(
+ team=team, pipeline=source, schema=schema, status=ExternalDataJobStatus.RUNNING, rows_synced=10
+ )
+
+ with (
+ patch.object(DataWarehouseTable, "get_columns", return_value={}),
+ patch.object(DataWarehouseTable, "get_count", return_value=100),
+ ):
+ async_to_sync(register_cdc_companion_table)(
+ run_id=str(job.id),
+ team_id=team.pk,
+ schema_id=schema.id,
+ resource_name=resource_name,
+ row_count=100,
+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,
+ queryable_folder=f"{resource_name}__query_a",
+ table_schema_dict={"id": "Int64", validity.valid_from: "DateTime", validity.valid_to: "DateTime"},
+ )
+
+ assert history_naming(ExternalDataSchema.objects.get(id=schema.id)) == expected
+
+ def test_does_not_take_the_record_of_a_source_table_with_the_history_name(self, team):
+ source = ExternalDataSource.objects.create(
+ source_id=str(uuid.uuid4()), connection_id=str(uuid.uuid4()), team=team, source_type="Postgres"
+ )
+ schema = ExternalDataSchema.objects.create(name="orders", team=team, source=source)
+ sibling_table = DataWarehouseTable.objects.create(
+ team=team,
+ name=build_table_name(source, "orders_cdc"),
+ format=DataWarehouseTableFormat.DeltaS3Wrapper,
+ url_pattern="s3://bucket/orders_cdc__query_a",
+ external_data_source=source,
+ )
+ ExternalDataSchema.objects.create(name="orders_cdc", team=team, source=source, table=sibling_table)
+ job = ExternalDataJob.objects.create(
+ team=team, pipeline=source, schema=schema, status=ExternalDataJobStatus.RUNNING, rows_synced=10
+ )
+
+ with pytest.raises(HistoryTableNameTakenError):
+ async_to_sync(register_cdc_companion_table)(
+ run_id=str(job.id),
+ team_id=team.pk,
+ schema_id=schema.id,
+ resource_name="orders_cdc",
+ row_count=100,
+ table_format=DataWarehouseTableFormat.DeltaS3Wrapper,
+ queryable_folder="orders_cdc__query_b",
+ )
+
+ sibling_table.refresh_from_db()
+ assert (sibling_table.url_pattern, sibling_table.is_history_table) == ("s3://bucket/orders_cdc__query_a", False)
+
def test_records_the_companion_size_on_the_companion_table(self, team):
# In `both` mode the schema's table is the snapshot table. The companion size must land on
# the companion's own record, for a new record and for an existing one.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py b/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py
index 92441c8bafdf..b612a0242387 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/schema_flags.py
@@ -17,6 +17,7 @@ from posthog.temporal.common.utils import retry_on_db_connection_drop
if TYPE_CHECKING:
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
+ from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
def is_schema_flag_enabled(schema: ExternalDataSchema, flag: str) -> bool:
@@ -26,10 +27,25 @@ def is_schema_flag_enabled(schema: ExternalDataSchema, flag: str) -> bool:
released to a single table — set a release condition `schema_id = <id>` to dogfood a feature
on one schema before rolling out by team/org/project.
"""
+ return _is_team_flag_enabled(
+ schema.team_id,
+ flag,
+ {"schema_id": str(schema.id), "team_id": str(schema.team_id), "source_type": schema.source.source_type},
+ )
+
+
+def is_source_flag_enabled(source: ExternalDataSource, flag: str) -> bool:
+ """Evaluate a rollout flag for a whole source, for a decision made before its schemas exist."""
+ return _is_team_flag_enabled(
+ source.team_id, flag, {"team_id": str(source.team_id), "source_type": source.source_type}
+ )
+
+
+def _is_team_flag_enabled(team_id: int, flag: str, person_properties: dict[str, str]) -> bool:
from posthog.models import Team # noqa: PLC0415 — keeps this module importable before app registry setup
try:
- team = retry_on_db_connection_drop(lambda: Team.objects.only("uuid", "organization_id").get(id=schema.team_id))
+ team = retry_on_db_connection_drop(lambda: Team.objects.only("uuid", "organization_id").get(id=team_id))
except Team.DoesNotExist:
return False
except (OperationalError, InterfaceError) as e:
@@ -45,11 +61,7 @@ def is_schema_flag_enabled(schema: ExternalDataSchema, flag: str) -> bool:
flag,
str(team.uuid),
groups={"organization": str(team.organization_id), "project": str(team.id)},
- person_properties={
- "schema_id": str(schema.id),
- "team_id": str(schema.team_id),
- "source_type": schema.source.source_type,
- },
+ person_properties=person_properties,
group_properties={
"organization": {"id": str(team.organization_id)},
"project": {"id": str(team.id)},
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index f57857ee3e23..36fda6b2a774 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -845,6 +845,10 @@ class PostgresSource(
# parent) — deterministic, so don't retry. `XminUnsupportedError` matches once Temporal
# wraps the failure; the message fragment matches the raw activity-level `str(e)`.
"XminUnsupportedError": None,
+ "is already the name of another table of this source": (
+ "The history table for this table would have the same name as another table of this source. "
+ "Choose the consolidated table only for this table, or rename the other table on your database."
+ ),
"xmin replication": None,
"NoSuchTableError": None,
"is not permitted to log in": _ROLE_CANNOT_LOG_IN_ERROR,
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index d2620a3360b8..67bccb2a624c 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -63,6 +63,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.sharepoint
from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.source import StripeSource
from products.warehouse_sources.backend.tests.api.utils import create_external_data_source_ok
+_PH_HISTORY_NAMING_FLAG = (
+ "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"
+)
+
pytestmark = [
pytest.mark.django_db,
]
@@ -1358,13 +1362,13 @@ class TestExternalDataSchema(APIBaseTest):
{
"primary_key_columns": ["id"],
"cdc_table_mode": "both",
- "schema_metadata": {"columns": [{"name": "id"}, {"name": "valid_from"}]},
+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "_ph_valid_from"}]},
},
None,
status.HTTP_400_BAD_REQUEST,
ExternalDataSchema.SyncType.FULL_REFRESH,
None,
- "valid_from",
+ "_ph_valid_from",
),
(
"accepts_a_validity_column_name_without_a_history_table",
@@ -4599,7 +4603,6 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
[
(
"turning_history_on_over_a_synced_column_is_refused",
- "valid_to",
"consolidated",
None,
{"cdc_table_mode": "both"},
@@ -4607,7 +4610,6 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
),
(
"turning_history_on_over_a_column_left_out_is_allowed",
- "valid_to",
"consolidated",
["id"],
{"cdc_table_mode": "both"},
@@ -4615,52 +4617,75 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
),
(
"adding_the_column_to_enabled_columns_is_refused",
- "valid_to",
"both",
["id"],
{"enabled_columns": ["id", "valid_to"]},
True,
),
+ ("editing_a_table_that_already_syncs_it_is_allowed", "both", None, {"enabled_columns": None}, False),
(
- "editing_a_table_that_already_syncs_it_is_allowed",
+ "a_new_history_table_leaves_valid_to_to_the_source",
+ "consolidated",
+ None,
+ {"cdc_table_mode": "both"},
+ False,
"valid_to",
- "both",
+ False,
+ ),
+ (
+ "a_new_history_table_refuses_its_own_validity_name",
+ "consolidated",
None,
- {"enabled_columns": None},
+ {"cdc_table_mode": "both"},
+ True,
+ "_ph_valid_to",
False,
),
# Capture columns are refused on every CDC table, history or not.
(
"adding_a_capture_column_to_enabled_columns_is_refused",
- "_PH_DELETED",
"consolidated",
["id"],
{"enabled_columns": ["id", "_PH_DELETED"]},
True,
+ "_PH_DELETED",
+ False,
),
(
"editing_a_table_that_already_syncs_a_capture_column_is_allowed",
- "_PH_DELETED",
"consolidated",
None,
{"enabled_columns": None},
False,
+ "_PH_DELETED",
+ False,
),
]
)
+ @mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)
def test_patch_checks_the_columns_a_cdc_sync_writes(
self,
_name: str,
- column: str,
initial_mode: str,
initial_enabled_columns: list[str] | None,
request_body: dict,
refused: bool,
+ column: str = "valid_to",
+ legacy_table: bool = True,
):
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
ExternalDataSchemaSerializer,
)
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name
+ if legacy_table:
+ DataWarehouseTable.objects.create(
+ team=self.team,
+ name=build_table_name(self.source, f"{self.schema.name}_cdc"),
+ format="DeltaS3Wrapper",
+ url_pattern="s3://bucket/orders_cdc",
+ external_data_source=self.source,
+ )
self.schema.sync_type_config["cdc_table_mode"] = initial_mode
self.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": column}]}
self.schema.enabled_columns = initial_enabled_columns
@@ -4690,6 +4715,81 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
assert self.schema.enabled_columns == (
initial_enabled_columns if refused else request_body.get("enabled_columns", initial_enabled_columns)
)
+ keeps_history = not refused and self.schema.cdc_table_mode in ("cdc_only", "both")
+ expected_pin = ("legacy" if legacy_table else "ph_validity") if keeps_history else None
+ assert self.schema.sync_type_config.get("history_naming") == expected_pin
+
+ def _save_through_serializer(self, schema: ExternalDataSchema, request_body: dict) -> None:
+ from products.warehouse_sources.backend.presentation.views.external_data_schema import (
+ ExternalDataSchemaSerializer,
+ )
+
+ serializer = ExternalDataSchemaSerializer(
+ schema, data=request_body, partial=True, context={"team_id": self.team.pk, "post_commit_actions": []}
+ )
+ with mock.patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_schema.is_any_external_data_schema_paused",
+ return_value=False,
+ ):
+ serializer.is_valid(raise_exception=True)
+ serializer.save()
+
+ @mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)
+ def test_patch_refuses_a_history_table_named_like_a_synced_table(self):
+ ExternalDataSchema.objects.create(
+ name="public.orders_history", team=self.team, source=self.source, should_sync=True
+ )
+
+ with pytest.raises(ValidationError, match="share its name"):
+ self._save_through_serializer(self.schema, {"cdc_table_mode": "both"})
+
+ self.schema.refresh_from_db()
+ assert self.schema.cdc_table_mode == "consolidated"
+
+ @mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)
+ def test_patch_refuses_syncing_a_table_named_like_another_schemas_history_table(self):
+ # The history table is not registered until the first snapshot loads, and the name is taken already.
+ self.schema.sync_type_config["cdc_table_mode"] = "both"
+ self.schema.save()
+ sibling = ExternalDataSchema.objects.create(
+ name="public.orders_history",
+ team=self.team,
+ source=self.source,
+ should_sync=False,
+ sync_type=ExternalDataSchema.SyncType.FULL_REFRESH,
+ )
+
+ with pytest.raises(ValidationError, match="history table of another table"):
+ self._save_through_serializer(sibling, {"should_sync": True})
+
+ sibling.refresh_from_db()
+ assert sibling.should_sync is False
+
+ @parameterized.expand(
+ [
+ ("no_history_table", ExternalDataSchema.SyncType.CDC, "consolidated", False, None),
+ ("legacy_naming", ExternalDataSchema.SyncType.CDC, "both", False, "public.orders_cdc"),
+ ("new_naming", ExternalDataSchema.SyncType.CDC, "both", True, "public.orders_history"),
+ # Leaving CDC keeps the stored table mode, but the table no longer keeps history.
+ ("left_cdc", ExternalDataSchema.SyncType.INCREMENTAL, "both", True, None),
+ ]
+ )
+ def test_history_table_name_names_the_table_the_mode_writes(
+ self, _name: str, sync_type: str, mode: str, ph_naming: bool, storage_name: str | None
+ ):
+ from products.warehouse_sources.backend.presentation.views.external_data_schema import (
+ ExternalDataSchemaSerializer,
+ )
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name
+
+ self.schema.sync_type = sync_type
+ self.schema.sync_type_config["cdc_table_mode"] = mode
+ self.schema.save()
+
+ with mock.patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: ph_naming):
+ name = ExternalDataSchemaSerializer().get_history_table_name(self.schema)
+
+ assert name == (None if storage_name is None else build_table_name(self.source, storage_name))
def test_patch_does_not_revert_a_concurrent_delete_data(self):
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 56477cc0e485..a8d2add0c3cd 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -141,6 +141,10 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.stripe.sou
from products.warehouse_sources.backend.temporal.data_imports.sources.tiktok_ads.utils import TikTokAdsAPIError
from products.warehouse_sources.backend.temporal.data_imports.sources.trino.source import TrinoSource
+_PH_HISTORY_NAMING_FLAG = (
+ "products.warehouse_sources.backend.temporal.data_imports.cdc.history_table.is_source_flag_enabled"
+)
+
def _configure_source_mock_versioning(mock_get_source) -> None:
"""Tests that patch `SourceRegistry.get_source` with a bare MagicMock must give the versioning
@@ -3404,6 +3408,7 @@ class TestExternalDataSource(APIBaseTest):
"row_filter_columns": None,
"available_columns": [],
"source_column_metadata_available": False,
+ "history_table_name": None,
"source": None,
"api_version": None,
"api_version_deprecation": None,
@@ -5004,6 +5009,7 @@ class TestExternalDataSource(APIBaseTest):
assert enabled.sync_type == ExternalDataSchema.SyncType.CDC
assert enabled.sync_type_config["cdc_mode"] == "snapshot"
assert enabled.sync_type_config["cdc_table_mode"] == (events_table_mode or "consolidated")
+ assert enabled.sync_type_config.get("history_naming") == ("legacy" if expected_history_modes else None)
history_events = [
c.kwargs["properties"]
for c in mock_capture.call_args_list
@@ -5027,7 +5033,7 @@ class TestExternalDataSource(APIBaseTest):
@parameterized.expand(
[
- ("no_primary_key", [("id", "uuid", False)], {}, "primary key", "consolidated", None),
+ ("no_primary_key", [("id", "uuid", False)], {}, "primary key", "consolidated", None, None),
(
"reserved_column",
[("id", "uuid", False), ("_ph_cdc_seq", "bigint", True)],
@@ -5035,6 +5041,7 @@ class TestExternalDataSource(APIBaseTest):
"_ph_cdc_seq",
"consolidated",
None,
+ None,
),
(
"capture_column_the_load_renames",
@@ -5043,22 +5050,34 @@ class TestExternalDataSource(APIBaseTest):
"tracking_link (_ph_deleted)",
"consolidated",
None,
+ None,
),
(
"validity_column_with_a_history_table",
- [("id", "uuid", False), ("valid_to", "timestamp", True)],
+ [("id", "uuid", False), ("_ph_valid_to", "timestamp", True)],
{"tracking_link": ["id"]},
- "valid_to",
+ "_ph_valid_to",
"cdc_only",
None,
+ None,
),
(
"validity_column_in_enabled_columns_that_hold_a_non_string",
- [("id", "uuid", False), ("valid_to", "timestamp", True)],
+ [("id", "uuid", False), ("_ph_valid_to", "timestamp", True)],
{"tracking_link": ["id"]},
- "valid_to",
+ "_ph_valid_to",
"cdc_only",
- ["id", "valid_to", {}],
+ ["id", "_ph_valid_to", {}],
+ None,
+ ),
+ (
+ "history_table_named_like_another_synced_table",
+ [("id", "uuid", False)],
+ {"tracking_link": ["id"]},
+ "same name",
+ "both",
+ None,
+ "tracking_link_history",
),
]
)
@@ -5075,6 +5094,7 @@ class TestExternalDataSource(APIBaseTest):
@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.get_primary_key_columns")
@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.cdc_pg_connection")
@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.SourceRegistry.get_source")
+ @patch(_PH_HISTORY_NAMING_FLAG, new=lambda *_: True)
def test_create_postgres_cdc_rejects_a_table_it_cannot_capture(
self,
_name,
@@ -5083,6 +5103,7 @@ class TestExternalDataSource(APIBaseTest):
expected_in_message,
cdc_table_mode,
enabled_columns,
+ sibling_table,
mock_get_source,
mock_cdc_pg_connection,
mock_get_primary_key_columns,
@@ -5119,7 +5140,23 @@ class TestExternalDataSource(APIBaseTest):
foreign_keys=[],
source_schema="public",
source_table_name="tracking_link",
- )
+ ),
+ *(
+ [
+ SourceSchema(
+ name=sibling_table,
+ supports_incremental=False,
+ supports_append=False,
+ supports_cdc=False,
+ columns=[("id", "uuid", False)],
+ foreign_keys=[],
+ source_schema="public",
+ source_table_name=sibling_table,
+ )
+ ]
+ if sibling_table
+ else []
+ ),
]
mock_cdc_pg_connection.return_value.__enter__.return_value = object()
@@ -5147,6 +5184,11 @@ class TestExternalDataSource(APIBaseTest):
"cdc_table_mode": cdc_table_mode,
**({"enabled_columns": enabled_columns} if enabled_columns is not None else {}),
},
+ *(
+ [{"name": sibling_table, "should_sync": True, "sync_type": "full_refresh"}]
+ if sibling_table
+ else []
+ ),
],
},
},
@@ -12088,23 +12130,38 @@ class TestDisableCDC(APIBaseTest):
purged = {call.args[1] for call in mock_purge.call_args_list}
assert purged == {str(cdc_schema.id)} # CDC schemas only
+ @parameterized.expand(
+ [
+ # Registered before the history marker existed, so only its name says what it is.
+ ("legacy_table_without_the_marker", "orders_cdc", {}),
+ ("new_table", "orders_history", {"history_table": True, "history_naming": "ph_validity"}),
+ ]
+ )
@patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.cleanup_resources",
return_value=None,
)
- def test_disable_cdc_soft_deletes_companion_cdc_tables(self, _cleanup) -> None:
+ def test_disable_cdc_soft_deletes_companion_cdc_tables(
+ self, _name: str, storage_name: str, options: dict, _cleanup
+ ) -> None:
+ from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import build_table_name
+
source = _make_postgres_source(self.team.pk, self.user, cdc_enabled=True)
+ ExternalDataSchema.objects.create(
+ team_id=self.team.pk, name="orders", source=source, sync_type=ExternalDataSchema.SyncType.CDC
+ )
cdc_companion = DataWarehouseTable.objects.create(
team_id=self.team.pk,
- name="pg_orders_cdc",
+ name=build_table_name(source, storage_name),
external_data_source_id=source.pk,
format=DataWarehouseTable.TableFormat.DeltaS3Wrapper,
- url_pattern="s3://bucket/orders_cdc",
+ url_pattern=f"s3://bucket/{storage_name}",
+ options=options,
)
main_table = DataWarehouseTable.objects.create(
team_id=self.team.pk,
- name="pg_orders",
+ name=build_table_name(source, "orders"),
external_data_source_id=source.pk,
format=DataWarehouseTable.TableFormat.DeltaS3Wrapper,
url_pattern="s3://bucket/orders",
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index 7a0d245ef220..c4cba5e5575a 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -533,6 +533,11 @@ export interface ExternalDataSchemaApi {
readonly available_columns: readonly ExternalDataSchemaApiAvailableColumnsItem[]
/** Whether exact source-side column metadata is available for safe source-query projection. */
readonly source_column_metadata_available: boolean
+ /**
+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.
+ * @nullable
+ */
+ readonly history_table_name: string | null
/**
* Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.
* @nullable
@@ -723,6 +728,11 @@ export interface PatchedExternalDataSchemaApi {
readonly available_columns?: readonly PatchedExternalDataSchemaApiAvailableColumnsItem[]
/** Whether exact source-side column metadata is available for safe source-query projection. */
readonly source_column_metadata_available?: boolean
+ /**
+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.
+ * @nullable
+ */
+ readonly history_table_name?: string | null
/**
* Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.
* @nullable
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 303db48b942d..39f49a66d683 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -51824,6 +51824,11 @@ export namespace Schemas {
readonly available_columns: readonly ExternalDataSchemaAvailableColumnsItem[];
/** Whether exact source-side column metadata is available for safe source-query projection. */
readonly source_column_metadata_available: boolean;
+ /**
+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.
+ * @nullable
+ */
+ readonly history_table_name: string | null;
/**
* Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.
* @nullable
@@ -88018,6 +88023,11 @@ export namespace Schemas {
readonly available_columns?: readonly PatchedExternalDataSchemaAvailableColumnsItem[];
/** Whether exact source-side column metadata is available for safe source-query projection. */
readonly source_column_metadata_available?: boolean;
+ /**
+ * Name of the history table that keeps every version of each row, or `null` when the table keeps no history. A history table created before `_history` names existed keeps its `_cdc` name.
+ * @nullable
+ */
+ readonly history_table_name?: string | null;
/**
* Lightweight parent-source summary (id, source_type, access_method, column-selection support, the requesting user's access level). Only populated on the single-schema retrieve endpoint — `null` elsewhere — so read-only views can render without fetching the full source and all its schemas.
* @nullable