Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): hide capture columns on history tables (#116583)

Daniel RCc9b9b3492e30
diff --git a/products/data_warehouse/backend/models/test/test_table.py b/products/data_warehouse/backend/models/test/test_table.py
index 3ccb7e89a196..bd8f48e63942 100644
--- a/products/data_warehouse/backend/models/test/test_table.py
+++ b/products/data_warehouse/backend/models/test/test_table.py
@@ -629,6 +629,47 @@ class TestTable(BaseTest):
],
)
+ @parameterized.expand(
+ [
+ ("history_table", {"history_table": True}, {"id", "_ph_deleted", "valid_from", "valid_to"}),
+ (
+ "consolidated_table_whose_source_has_validity_columns",
+ {},
+ {
+ "id",
+ "_ph_deleted",
+ "valid_from",
+ "valid_to",
+ "_ph_cdc_op",
+ "_ph_cdc_seq",
+ "_ph_cdc_timestamp",
+ "_ph_deleted_at",
+ },
+ ),
+ ]
+ )
+ def test_history_table_hides_capture_bookkeeping_columns(
+ self, _name: str, options: dict, expected_visible: set[str]
+ ):
+ capture_columns = ["_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted", "_ph_deleted_at"]
+ table = DataWarehouseTable.objects.create(
+ name="orders",
+ url_pattern="https://bucket.s3.amazonaws.com/orders",
+ format=DataWarehouseTable.TableFormat.DeltaS3Wrapper,
+ team=self.team,
+ options=options,
+ columns={
+ name: {"clickhouse": "String", "hogql": "StringDatabaseField"}
+ for name in ["id", *capture_columns, "valid_from", "valid_to"]
+ },
+ )
+
+ fields = table.hogql_definition().fields
+ assert {name for name, field in fields.items() if not field.hidden} == expected_visible
+ assert {column["name"] for column in table.get_user_facing_columns()} == expected_visible
+ # A hidden column must still resolve, so a query saved before it was hidden keeps running.
+ assert set(capture_columns) <= set(fields)
+
def test_hogql_definition_new_style(self):
credential = DataWarehouseCredential.objects.create(access_key="test", access_secret="test", team=self.team)
table = DataWarehouseTable.objects.create(
diff --git a/products/data_warehouse/backend/presentation/views/table.py b/products/data_warehouse/backend/presentation/views/table.py
index 556edd64767e..e3ec336f0e14 100644
--- a/products/data_warehouse/backend/presentation/views/table.py
+++ b/products/data_warehouse/backend/presentation/views/table.py
@@ -673,8 +673,12 @@ class TableViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viewsets.M
credential.save()
old_csv_allow_double_quotes = instance.csv_allow_double_quotes
+ was_history_table = instance.is_history_table
for attr, value in validated_data.items():
setattr(instance, attr, value)
+ # The sync owns this marker, and an `options` edit through this API must not drop it.
+ if was_history_table:
+ instance.mark_as_history_table()
if (
instance._is_csv_format()
diff --git a/products/data_warehouse/backend/tests/api/test_table.py b/products/data_warehouse/backend/tests/api/test_table.py
index 1caec0c37956..60c569c17fb9 100644
--- a/products/data_warehouse/backend/tests/api/test_table.py
+++ b/products/data_warehouse/backend/tests/api/test_table.py
@@ -796,6 +796,24 @@ class TestTable(APIBaseTest):
assert response.status_code == 200
assert response.json()["name"] == "test_table2"
+ def test_update_options_keeps_the_history_table_marker(self):
+ table = DataWarehouseTable.objects.create(
+ name="orders_cdc",
+ format="Parquet",
+ team=self.team,
+ team_id=self.team.pk,
+ columns={},
+ options={"history_table": True},
+ )
+ response = self.client.patch(
+ f"/api/projects/{self.team.id}/warehouse_tables/{table.id}",
+ {"options": {}},
+ format="json",
+ )
+ assert response.status_code == 200, response.content
+ table.refresh_from_db()
+ assert table.is_history_table
+
def test_update_table_url_pattern_to_internal_bucket(self):
table = DataWarehouseTable.objects.create(
name="test_table",
diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.py
index 20a1dd4d42b0..32e72aa4ae61 100644
--- a/products/warehouse_sources/backend/facade/api.py
+++ b/products/warehouse_sources/backend/facade/api.py
@@ -49,10 +49,7 @@ from products.warehouse_sources.backend.models.external_data_job import (
)
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema as _ExternalDataSchema
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource as _ExternalDataSource
-from products.warehouse_sources.backend.models.table import (
- HIDDEN_COLUMNS,
- DataWarehouseTable as _DataWarehouseTable,
-)
+from products.warehouse_sources.backend.models.table import DataWarehouseTable as _DataWarehouseTable
# Framework-free helper transforms — re-exported as the public helper surface.
from products.warehouse_sources.backend.models.util import (
@@ -610,6 +607,7 @@ def all_queryable_table_columns(team_id: int, table_ids: Collection[UUID] | None
"id",
"name",
"columns",
+ "options",
"external_data_source_id",
"external_data_source__id",
"external_data_source__prefix",
@@ -628,10 +626,11 @@ def _hogql_visible_columns(table: _DataWarehouseTable) -> dict[str, str]:
# The same raw -> visible mapping DataWarehouseTable.get_user_facing_columns applies, keeping
# the stored type rather than the cleaned one so a consumer still reads any Nullable() wrapper.
hogql_by_raw = get_hogql_column_name_mapping(table.table_name_without_prefix())
+ hidden = table.hidden_columns()
return {
hogql_by_raw.get(name, name): type_
for name, entry in (table.columns or {}).items()
- if name not in HIDDEN_COLUMNS and (type_ := _clickhouse_column_type(entry)) is not None
+ if name not in hidden and (type_ := _clickhouse_column_type(entry)) is not None
}
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.py
index f3ef8ef4d5e4..d4d5457ff0a3 100644
--- a/products/warehouse_sources/backend/facade/source_management.py
+++ b/products/warehouse_sources/backend/facade/source_management.py
@@ -29,6 +29,8 @@ _LAZY = {
"tables_wait_for_repair": "cdc.broken",
"purge_buffer_prefix": "cdc.buffer",
"CDC_SEQ_COLUMN": "cdc.batcher",
+ "is_history_table_mode": "cdc.batcher",
+ "history_column_collisions": "cdc.batcher",
"BUFFER_LANE": "cdc.snapshot_lane",
"CDC_RESET_PENDING_KEY": "cdc.snapshot_lane",
"hand_reset_to_capture_if_sync_running": "cdc.snapshot_lane",
diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.py
index 674a0218a469..5bc8bb2efdea 100644
--- a/products/warehouse_sources/backend/models/table.py
+++ b/products/warehouse_sources/backend/models/table.py
@@ -187,6 +187,15 @@ type DataWarehouseTableIntrospectedColumns = dict[str, DataWarehouseTableIntrosp
# and never user-facing.
HIDDEN_COLUMNS: frozenset[str] = frozenset({"_dlt_id", "_dlt_load_id", "_ph_debug", PARTITION_KEY})
+# Capture bookkeeping that a history table carries next to `valid_from`, `valid_to` and `_ph_deleted`,
+# which already tell a reader when a version held and whether it is a delete. The names mirror the
+# constants in `cdc/batcher.py`, which this module does not import because that pulls in pyarrow.
+HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset({"_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted_at"})
+# Set in `options` when the sync registers a table as a schema's history table. Column names cannot
+# tell the two apart, because a source table can have its own `valid_from` and `valid_to` columns.
+HISTORY_TABLE_OPTION = "history_table"
+
+
# Characters that carry quoting meaning once a column name is written into SQL. The structure
# builder escapes them already, so this is a second layer: it keeps such a name out of the other
# sinks that read this column store, not all of which escape as thoroughly (for example
@@ -603,6 +612,19 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
prefix = ""
return self.name[len(prefix) :]
+ @property
+ def is_history_table(self) -> bool:
+ return (self.options or {}).get(HISTORY_TABLE_OPTION) is True
+
+ def mark_as_history_table(self) -> None:
+ self.options = {**(self.options or {}), HISTORY_TABLE_OPTION: True}
+
+ def hidden_columns(self) -> frozenset[str]:
+ """The plumbing columns to keep out of the catalog for this table."""
+ if self.is_history_table:
+ return HIDDEN_COLUMNS | HISTORY_HIDDEN_COLUMNS
+ return HIDDEN_COLUMNS
+
def get_user_facing_columns(self) -> list[dict[str, Any]]:
"""Synced columns as `[{name, data_type, is_nullable}]`, skipping internal plumbing columns.
@@ -618,9 +640,10 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
`get_hogql_column_name_mapping(self.table_name_without_prefix())` directly.
"""
hogql_by_raw = get_hogql_column_name_mapping(self.table_name_without_prefix())
+ hidden = self.hidden_columns()
result: list[dict[str, Any]] = []
for name, definition in (self.columns or {}).items():
- if name in HIDDEN_COLUMNS:
+ if name in hidden:
continue
if isinstance(definition, dict):
clickhouse_type = definition.get("clickhouse") or definition.get("hogql") or ""
@@ -1298,6 +1321,11 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet
if fields.get(PARTITION_KEY):
del fields[PARTITION_KEY]
fields = {**fields, **default_fields}
+ # Hidden rather than removed, so a saved query that names one of these still resolves.
+ if self.is_history_table:
+ for name in HISTORY_HIDDEN_COLUMNS:
+ if (field := fields.get(name)) is not None:
+ fields[name] = field.model_copy(update={"hidden": True})
table_def = HogQLDataWarehouseTable(
name=self.name,
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index cdc02a3eaa96..55f1dc54fa8f 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -74,6 +74,8 @@ from products.warehouse_sources.backend.facade.source_management import (
filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,
get_cdc_adapter,
hand_reset_to_capture_if_sync_running,
+ history_column_collisions,
+ is_history_table_mode,
purge_buffer_prefix,
repair_is_running,
resnapshot_stays_in_buffer,
@@ -444,20 +446,55 @@ def _apply_primary_key_columns(
)
-def _refuse_reserved_cdc_column(instance: ExternalDataSchema) -> None:
- """Capture stamps each change with this column, so a source column of the same name would fail
- the source's sync. Checked against the columns discovery recorded, which are the source's own."""
+def _source_column_names(instance: ExternalDataSchema) -> set[str]:
+ """The columns discovery recorded, which are the source's own."""
metadata = instance.schema_metadata or {}
columns = metadata.get("columns") if isinstance(metadata, dict) else None
- if isinstance(columns, list) and any(
- isinstance(column, dict) and column.get("name") == CDC_SEQ_COLUMN for column in columns
- ):
+ if not isinstance(columns, list):
+ return set()
+ return {column["name"] for column in columns if isinstance(column, dict) and isinstance(column.get("name"), str)}
+
+
+def _refuse_reserved_cdc_column(instance: ExternalDataSchema) -> None:
+ """Capture stamps each change with this column, so a source column of the same name would fail
+ the source's sync."""
+ if CDC_SEQ_COLUMN in _source_column_names(instance):
raise ValidationError(
"Change data capture can't sync a column named _ph_cdc_seq, because PostHog uses that name. "
"Rename the column on your database, or choose another sync method for this table."
)
+def _refuse_reserved_history_columns(
+ instance: ExternalDataSchema,
+ *,
+ previous_mode: str | None,
+ previous_enabled_columns: list[str] | None,
+ previous_primary_keys: list[str] | None,
+ new_mode: str | None,
+ new_enabled_columns: list[str] | None,
+ new_primary_keys: list[str] | None,
+) -> None:
+ """The history table stamps each row version with `valid_from` and `valid_to`, so a synced source
+ column of the same name fails every load into it. Only a collision this request adds is refused,
+ so a table that already has one can still be edited or switched back."""
+ if not is_history_table_mode(new_mode):
+ return
+ columns = _source_column_names(instance)
+ taken = history_column_collisions(columns, new_enabled_columns, new_primary_keys, instance.incremental_field)
+ if instance.sync_type == ExternalDataSchema.SyncType.CDC and is_history_table_mode(previous_mode):
+ existing = history_column_collisions(
+ columns, previous_enabled_columns, previous_primary_keys, instance.incremental_field
+ )
+ taken = [column for column in taken if column not in existing]
+ if taken:
+ raise ValidationError(
+ f"A history table can't store the column {' or '.join(taken)}, because PostHog uses valid_from and "
+ "valid_to to record when each version was current. Rename the column on your database, or choose "
+ "the consolidated table only for this table."
+ )
+
+
def schema_display_status(schema: ExternalDataSchema) -> str | None:
"""The user-facing sync status, mapping the two billing-limit statuses to friendly labels.
Shared by the full and list schema serializers so the labels stay identical."""
@@ -1318,6 +1355,20 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
payload["cdc_table_mode"] = cdc_table_mode
validated_data["sync_type_config"] = payload
+ # Checked once the branches above settle the mode and the key, because `enabled_columns` alone
+ # can also bring a validity column into the history table.
+ if (sync_type if "sync_type" in data else instance.sync_type) == ExternalDataSchema.SyncType.CDC:
+ resulting_config = validated_data.get("sync_type_config", instance.sync_type_config) or {}
+ _refuse_reserved_history_columns(
+ instance,
+ previous_mode=previous_cdc_table_mode,
+ previous_enabled_columns=instance.enabled_columns,
+ previous_primary_keys=original_sync_type_config.get("primary_key_columns"),
+ new_mode=resulting_config.get("cdc_table_mode"),
+ new_enabled_columns=validated_data.get("enabled_columns", instance.enabled_columns),
+ new_primary_keys=resulting_config.get("primary_key_columns"),
+ )
+
should_sync = validated_data.get("should_sync", None)
sync_frequency = data.get("sync_frequency", None)
sync_time_of_day_in_payload = "sync_time_of_day" in data
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
index 79630b5eef02..9313df03f593 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py
@@ -69,6 +69,8 @@ from products.warehouse_sources.backend.facade.source_management import (
fetch_docs_text,
filter_dwh_columns_by_enabled_columns,
get_cdc_adapter,
+ history_column_collisions,
+ is_history_table_mode,
is_team_allowlisted_for_internal_hosts,
new_source_requires_ssl,
sql_schema_metadata,
@@ -93,6 +95,14 @@ from products.warehouse_sources.backend.presentation.views.source_api_versions i
from . import base, connection_options, credential_store, helpers, webhook_setup
+def _payload_enabled_columns(schema: dict[str, Any]) -> list[str] | None:
+ """`None` syncs every column. `[]` syncs only the always-retained key and incremental field."""
+ payload_enabled_columns = schema.get("enabled_columns")
+ if not isinstance(payload_enabled_columns, list):
+ return None
+ return [str(column) for column in payload_enabled_columns if isinstance(column, str)]
+
+
class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializers.ModelSerializer):
account_id = serializers.CharField(write_only=True)
client_secret = serializers.CharField(write_only=True)
@@ -1451,6 +1461,37 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
},
)
+ # The history table stamps each row version with these columns, so a synced source column
+ # of the same name would fail every load into it.
+ tables_with_validity_column = sorted(
+ {
+ schema["name"]
+ for schema in payload_schemas
+ if schema.get("sync_type") == "cdc"
+ and schema.get("should_sync", False)
+ and is_history_table_mode(schema.get("cdc_table_mode"))
+ and isinstance(schema.get("name"), str)
+ and history_column_collisions(
+ [column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],
+ _payload_enabled_columns(schema),
+ pk_columns_by_table.get(schema["name"]),
+ )
+ }
+ )
+ if tables_with_validity_column:
+ new_source_model.delete()
+ return Response(
+ status=status.HTTP_400_BAD_REQUEST,
+ data={
+ "message": (
+ "A history table can't store a column that loads as valid_from or valid_to, because "
+ "PostHog uses those names to record when each version was current. Rename the column on "
+ "your database, or choose the consolidated table only for these tables: "
+ f"{', '.join(tables_with_validity_column)}."
+ )
+ },
+ )
+
# Engine-side CDC resource setup runs after PK validation so we don't leave
# replication state on the source for a config we're about to refuse.
if cdc_enabled:
@@ -1476,15 +1517,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):
primary_key_columns = schema.get("primary_key_columns")
sync_time_of_day = schema.get("sync_time_of_day")
should_sync = schema.get("should_sync", False)
- payload_enabled_columns = schema.get("enabled_columns")
- if isinstance(payload_enabled_columns, list):
- # `[]` and `None` are distinct: `None` means sync all columns, `[]` means
- # sync only the always-retained PK + incremental field.
- enabled_columns: list[str] | None = [
- str(column) for column in payload_enabled_columns if isinstance(column, str)
- ]
- else:
- enabled_columns = None
+ enabled_columns = _payload_enabled_columns(schema)
payload_row_filters = schema.get("row_filters")
row_filters: list[dict[str, Any]] | None = (
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
index ce0c881c2486..278a583313ad 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py
@@ -1,12 +1,13 @@
from __future__ import annotations
from collections import defaultdict
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
import pyarrow as pa
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnError
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
# CDC metadata column names — database-agnostic
CDC_OP_COLUMN = "_ph_cdc_op"
@@ -47,6 +48,33 @@ TOAST_OMITTED_COLUMN = "_ph_toast_omitted"
SCD2_VALID_FROM_COLUMN = "valid_from"
SCD2_VALID_TO_COLUMN = "valid_to"
+# Values of `cdc_table_mode` that write the history table.
+CDC_HISTORY_TABLE_MODES: frozenset[str] = frozenset({"cdc_only", "both"})
+
+
+def is_history_table_mode(cdc_table_mode: object) -> bool:
+ # The create payload is free-form, so a malformed mode can be stored, and a list or dict is unhashable.
+ return isinstance(cdc_table_mode, str) and cdc_table_mode in CDC_HISTORY_TABLE_MODES
+
+
+def history_column_collisions(
+ column_names: Iterable[str],
+ enabled_columns: list[str] | None,
+ primary_keys: list[str] | None,
+ incremental_field: str | None = None,
+) -> list[str]:
+ """Source columns that the history table would receive under its validity column names."""
+ synced = set(column_names)
+ # The same projection capture applies (`filter_columns_by_enabled_columns`), kept inline because
+ # that module's package imports Django models and this one sits on the capture path.
+ if enabled_columns is not None:
+ synced &= {*enabled_columns, *(primary_keys or []), *([incremental_field] if incremental_field else [])}
+ # The load renames columns to snake_case before the history transform adds the validity columns,
+ # so `validFrom` collides too. The source names are returned, because those are what a user renames.
+ validity_columns = {SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN}
+ return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in validity_columns)
+
+
# Columns that are CDC/SCD2 metadata — always filled from the event itself,
# never overwritten when enriching DELETE rows with last-known source data.
_CDC_METADATA_COLUMNS: frozenset[str] = frozenset(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
index 4d05f38c9086..ce94cc745109 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py
@@ -22,6 +22,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import
build_scd2_table,
enrich_delete_rows,
enrich_toast_omitted_rows,
+ history_column_collisions,
+ is_history_table_mode,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnError
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
@@ -711,6 +713,41 @@ class TestScd2TimestampType:
assert result.column(SCD2_VALID_TO_COLUMN).to_pylist()[0] is not None
+class TestHistoryColumnCollisions:
+ @parameterized.expand(
+ [
+ ("exact_name", ["id", "valid_to"], None, None, ["valid_to"]),
+ ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], None, None, ["VALID_TO", "validFrom"]),
+ ("left_out_of_enabled_columns", ["id", "validFrom"], ["id"], None, []),
+ ("kept_as_the_incremental_field", ["id", "valid_from"], [], "valid_from", ["valid_from"]),
+ ]
+ )
+ def test_names_the_source_columns_that_collide(
+ self,
+ _name: str,
+ column_names: list[str],
+ enabled_columns: list[str] | None,
+ incremental_field: str | None,
+ expected: list[str],
+ ) -> None:
+ assert history_column_collisions(column_names, enabled_columns, ["id"], incremental_field) == expected
+
+
+class TestIsHistoryTableMode:
+ @parameterized.expand(
+ [
+ ("cdc_only", "cdc_only", True),
+ ("both", "both", True),
+ ("consolidated", "consolidated", False),
+ ("missing", None, False),
+ ("list", [], False),
+ ("dict", {}, False),
+ ]
+ )
+ def test_reads_only_a_history_mode_string(self, _name: str, cdc_table_mode: object, expected: bool) -> None:
+ assert is_history_table_mode(cdc_table_mode) is expected
+
+
class TestScd2ReservedColumns:
@parameterized.expand([(SCD2_VALID_FROM_COLUMN,), (SCD2_VALID_TO_COLUMN,)])
def test_a_source_column_named_like_the_stamp_fails_before_the_writer(self, taken):
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
index 2c71e846636d..edb2a2906a3e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py
@@ -34,7 +34,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
update_sync_type_config_keys,
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
-from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable
from products.warehouse_sources.backend.models.util import hogql_type_name_for_clickhouse_type
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import companion_resource_name
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
@@ -699,6 +699,7 @@ async def register_cdc_companion_table(
table.format = table_format
table.url_pattern = new_url_pattern
table.queryable_folder = queryable_folder
+ table.mark_as_history_table()
_refresh_cumulative_row_count(table, logger, companion_table_name, live_row_count)
# Scope to the fields changed here so this out-of-transaction save doesn't rewrite
# `columns` with its pre-merge value before the column save below.
@@ -707,7 +708,7 @@ async def register_cdc_companion_table(
# on a fresh connection rather than let this escape as error-tracking noise.
# new_url_pattern above is derived from the job's own destination folder, not from
# request input, so this sync is a trusted writer of a credential-less table's URL.
- update_fields = ["format", "url_pattern", "queryable_folder", "row_count"]
+ update_fields = ["format", "url_pattern", "queryable_folder", "options", "row_count"]
if live_size_mib is not None:
table.size_in_s3_mib = live_size_mib
update_fields.append("size_in_s3_mib")
@@ -720,6 +721,7 @@ async def register_cdc_companion_table(
external_data_source_id=job.pipeline.id,
created_via=DataWarehouseTableCreatedVia.SOURCE,
size_in_s3_mib=live_size_mib,
+ options={HISTORY_TABLE_OPTION: True},
**table_params,
)
_record_storage_delta(job, storage_delta_mib)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
index 64ade129ac45..3439a609ac59 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py
@@ -22,7 +22,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
SelfManagedTableRef,
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
-from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTable
from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention
from products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import (
build_table_name,
@@ -406,6 +406,8 @@ class TestRegisterCdcCompanionTableStamp:
job.refresh_from_db()
assert companion.size_in_s3_mib == 5.0
assert job.storage_delta_mib == 5.0
+ # The catalog reads this marker to hide the capture columns.
+ assert companion.options[HISTORY_TABLE_OPTION] is True
# transaction=True: validate_schema_and_update_table writes to the DB from the async thread pool
diff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py
index aaf7e0f6f953..363be89fa294 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py
@@ -25,7 +25,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (
mark_schema_running_unless_halted,
)
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
-from products.warehouse_sources.backend.models.table import HIDDEN_COLUMNS, DataWarehouseTable
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
from products.warehouse_sources.backend.temporal.data_imports.destinations.enablement import (
NoActiveDestinationsError,
destination_ids_for_run,
@@ -141,7 +141,7 @@ def _enrichment_pending(team_id: int, table: DataWarehouseTable | None, schema:
return True
# Hidden plumbing columns (_dlt_id, partition key, …) are never enriched, so they'd otherwise
# show up perpetually in `columns - annotated` and keep this returning True on every sync.
- columns = set((table.columns or {}).keys()) - HIDDEN_COLUMNS
+ columns = set((table.columns or {}).keys()) - table.hidden_columns()
annotated = set(
WarehouseColumnAnnotation.objects.for_team(team_id)
.filter(table_id=table.id)
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
index 6f2da38fdba0..f70f4e4deb07 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py
@@ -17,6 +17,7 @@ import pytest_asyncio
from asgiref.sync import sync_to_async
from parameterized import parameterized
from rest_framework import status
+from rest_framework.exceptions import ValidationError
from temporalio.service import RPCError, RPCStatusCode
from posthog.api.test.test_organization import create_organization
@@ -1340,6 +1341,30 @@ class TestExternalDataSchema(APIBaseTest):
None,
"_ph_cdc_seq",
),
+ (
+ "rejects_a_history_table_for_a_source_column_named_like_a_validity_column",
+ {
+ "primary_key_columns": ["id"],
+ "cdc_table_mode": "both",
+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "valid_from"}]},
+ },
+ None,
+ status.HTTP_400_BAD_REQUEST,
+ ExternalDataSchema.SyncType.FULL_REFRESH,
+ None,
+ "valid_from",
+ ),
+ (
+ "accepts_a_validity_column_name_without_a_history_table",
+ {
+ "primary_key_columns": ["id"],
+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "valid_from"}]},
+ },
+ None,
+ status.HTTP_200_OK,
+ ExternalDataSchema.SyncType.CDC,
+ ["id"],
+ ),
]
)
def test_update_schema_to_cdc(
@@ -4503,6 +4528,74 @@ class TestSerializerLostUpdateProtection(APIBaseTest):
assert self.schema.cdc_table_mode == "both"
assert self.schema.sync_type_config["cdc_last_log_position"] == "0/900"
+ @parameterized.expand(
+ [
+ (
+ "turning_history_on_over_a_synced_column_is_refused",
+ "consolidated",
+ None,
+ {"cdc_table_mode": "both"},
+ True,
+ ),
+ (
+ "turning_history_on_over_a_column_left_out_is_allowed",
+ "consolidated",
+ ["id"],
+ {"cdc_table_mode": "both"},
+ False,
+ ),
+ (
+ "adding_the_column_to_enabled_columns_is_refused",
+ "both",
+ ["id"],
+ {"enabled_columns": ["id", "valid_to"]},
+ True,
+ ),
+ ("editing_a_table_that_already_syncs_it_is_allowed", "both", None, {"enabled_columns": None}, False),
+ ]
+ )
+ def test_patch_checks_the_validity_columns_the_history_table_receives(
+ self,
+ _name: str,
+ initial_mode: str,
+ initial_enabled_columns: list[str] | None,
+ request_body: dict,
+ refused: bool,
+ ):
+ from products.warehouse_sources.backend.presentation.views.external_data_schema import (
+ ExternalDataSchemaSerializer,
+ )
+
+ self.schema.sync_type_config["cdc_table_mode"] = initial_mode
+ self.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": "valid_to"}]}
+ self.schema.enabled_columns = initial_enabled_columns
+ self.schema.save()
+
+ serializer = ExternalDataSchemaSerializer(
+ self.schema,
+ data=request_body,
+ partial=True,
+ context={"team_id": self.team.pk, "post_commit_actions": []},
+ )
+ with mock.patch(
+ "products.warehouse_sources.backend.presentation.views.external_data_schema.is_any_external_data_schema_paused",
+ return_value=False,
+ ):
+ serializer.is_valid(raise_exception=True)
+ if refused:
+ with pytest.raises(ValidationError, match="valid_to"):
+ serializer.save()
+ else:
+ serializer.save()
+
+ self.schema.refresh_from_db()
+ assert self.schema.cdc_table_mode == (
+ initial_mode if refused else request_body.get("cdc_table_mode", initial_mode)
+ )
+ assert self.schema.enabled_columns == (
+ initial_enabled_columns if refused else request_body.get("enabled_columns", initial_enabled_columns)
+ )
+
def test_patch_does_not_revert_a_concurrent_delete_data(self):
from products.warehouse_sources.backend.presentation.views.external_data_schema import (
ExternalDataSchemaSerializer,
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index f413e9ce6327..57fd7be8869e 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -4974,12 +4974,30 @@ class TestExternalDataSource(APIBaseTest):
@parameterized.expand(
[
- ("no_primary_key", [("id", "uuid", False)], {}, "primary key"),
+ ("no_primary_key", [("id", "uuid", False)], {}, "primary key", "consolidated", None),
(
"reserved_column",
[("id", "uuid", False), ("_ph_cdc_seq", "bigint", True)],
{"tracking_link": ["id"]},
"_ph_cdc_seq",
+ "consolidated",
+ None,
+ ),
+ (
+ "validity_column_with_a_history_table",
+ [("id", "uuid", False), ("valid_to", "timestamp", True)],
+ {"tracking_link": ["id"]},
+ "valid_to",
+ "cdc_only",
+ None,
+ ),
+ (
+ "validity_column_in_enabled_columns_that_hold_a_non_string",
+ [("id", "uuid", False), ("valid_to", "timestamp", True)],
+ {"tracking_link": ["id"]},
+ "valid_to",
+ "cdc_only",
+ ["id", "valid_to", {}],
),
]
)
@@ -5002,6 +5020,8 @@ class TestExternalDataSource(APIBaseTest):
columns,
primary_keys,
expected_in_message,
+ cdc_table_mode,
+ enabled_columns,
mock_get_source,
mock_cdc_pg_connection,
mock_get_primary_key_columns,
@@ -5059,7 +5079,13 @@ class TestExternalDataSource(APIBaseTest):
"schema": "public",
"cdc_enabled": True,
"schemas": [
- {"name": "tracking_link", "should_sync": True, "sync_type": "cdc"},
+ {
+ "name": "tracking_link",
+ "should_sync": True,
+ "sync_type": "cdc",
+ "cdc_table_mode": cdc_table_mode,
+ **({"enabled_columns": enabled_columns} if enabled_columns is not None else {}),
+ },
],
},
},
diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.py
index df8b293fb814..3ea501642881 100644
--- a/products/warehouse_sources/backend/tests/test_table.py
+++ b/products/warehouse_sources/backend/tests/test_table.py
@@ -33,11 +33,18 @@ from posthog.models import Team
from products.warehouse_sources.backend.models.credential import DataWarehouseCredential
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.models.table import (
+ HISTORY_HIDDEN_COLUMNS,
DataWarehouseTable,
classify_warehouse_read_error,
get_hogql_field_for_column,
run_chdb_query,
)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ CDC_OP_COLUMN,
+ CDC_SEQ_COLUMN,
+ CDC_TIMESTAMP_COLUMN,
+ DELETED_AT_COLUMN,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (
TransientObjectStoreError,
)
@@ -719,6 +726,11 @@ class TestSchemaInferenceMode(BaseTest):
assert mock_sync_execute.call_count == 5
+class TestHistoryHiddenColumns(SimpleTestCase):
+ def test_names_match_the_columns_capture_writes(self) -> None:
+ assert HISTORY_HIDDEN_COLUMNS == {CDC_OP_COLUMN, CDC_SEQ_COLUMN, CDC_TIMESTAMP_COLUMN, DELETED_AT_COLUMN}
+
+
class TestGetHogqlFieldForColumn(SimpleTestCase):
@parameterized.expand(
[