:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): hide capture columns on history tables (#116583)
Daniel RCc9b9b3492e30
diff --git a/products/data_warehouse/backend/models/test/test_table.py b/products/data_warehouse/backend/models/test/test_table.pyindex 3ccb7e89a196..bd8f48e63942 100644--- a/products/data_warehouse/backend/models/test/test_table.py+++ b/products/data_warehouse/backend/models/test/test_table.py@@ -629,6 +629,47 @@ class TestTable(BaseTest):],)+ @parameterized.expand(+ [+ ("history_table", {"history_table": True}, {"id", "_ph_deleted", "valid_from", "valid_to"}),+ (+ "consolidated_table_whose_source_has_validity_columns",+ {},+ {+ "id",+ "_ph_deleted",+ "valid_from",+ "valid_to",+ "_ph_cdc_op",+ "_ph_cdc_seq",+ "_ph_cdc_timestamp",+ "_ph_deleted_at",+ },+ ),+ ]+ )+ def test_history_table_hides_capture_bookkeeping_columns(+ self, _name: str, options: dict, expected_visible: set[str]+ ):+ capture_columns = ["_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted", "_ph_deleted_at"]+ table = DataWarehouseTable.objects.create(+ name="orders",+ url_pattern="https://bucket.s3.amazonaws.com/orders",+ format=DataWarehouseTable.TableFormat.DeltaS3Wrapper,+ team=self.team,+ options=options,+ columns={+ name: {"clickhouse": "String", "hogql": "StringDatabaseField"}+ for name in ["id", *capture_columns, "valid_from", "valid_to"]+ },+ )++ fields = table.hogql_definition().fields+ assert {name for name, field in fields.items() if not field.hidden} == expected_visible+ assert {column["name"] for column in table.get_user_facing_columns()} == expected_visible+ # A hidden column must still resolve, so a query saved before it was hidden keeps running.+ assert set(capture_columns) <= set(fields)+def test_hogql_definition_new_style(self):credential = DataWarehouseCredential.objects.create(access_key="test", access_secret="test", team=self.team)table = DataWarehouseTable.objects.create(diff --git a/products/data_warehouse/backend/presentation/views/table.py b/products/data_warehouse/backend/presentation/views/table.pyindex 556edd64767e..e3ec336f0e14 100644--- a/products/data_warehouse/backend/presentation/views/table.py+++ b/products/data_warehouse/backend/presentation/views/table.py@@ -673,8 +673,12 @@ class TableViewSet(TeamAndOrgViewSetMixin, AccessControlViewSetMixin, viewsets.Mcredential.save()old_csv_allow_double_quotes = instance.csv_allow_double_quotes+ was_history_table = instance.is_history_tablefor attr, value in validated_data.items():setattr(instance, attr, value)+ # The sync owns this marker, and an `options` edit through this API must not drop it.+ if was_history_table:+ instance.mark_as_history_table()if (instance._is_csv_format()diff --git a/products/data_warehouse/backend/tests/api/test_table.py b/products/data_warehouse/backend/tests/api/test_table.pyindex 1caec0c37956..60c569c17fb9 100644--- a/products/data_warehouse/backend/tests/api/test_table.py+++ b/products/data_warehouse/backend/tests/api/test_table.py@@ -796,6 +796,24 @@ class TestTable(APIBaseTest):assert response.status_code == 200assert response.json()["name"] == "test_table2"+ def test_update_options_keeps_the_history_table_marker(self):+ table = DataWarehouseTable.objects.create(+ name="orders_cdc",+ format="Parquet",+ team=self.team,+ team_id=self.team.pk,+ columns={},+ options={"history_table": True},+ )+ response = self.client.patch(+ f"/api/projects/{self.team.id}/warehouse_tables/{table.id}",+ {"options": {}},+ format="json",+ )+ assert response.status_code == 200, response.content+ table.refresh_from_db()+ assert table.is_history_table+def test_update_table_url_pattern_to_internal_bucket(self):table = DataWarehouseTable.objects.create(name="test_table",diff --git a/products/warehouse_sources/backend/facade/api.py b/products/warehouse_sources/backend/facade/api.pyindex 20a1dd4d42b0..32e72aa4ae61 100644--- a/products/warehouse_sources/backend/facade/api.py+++ b/products/warehouse_sources/backend/facade/api.py@@ -49,10 +49,7 @@ from products.warehouse_sources.backend.models.external_data_job import ()from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema as _ExternalDataSchemafrom products.warehouse_sources.backend.models.external_data_source import ExternalDataSource as _ExternalDataSource-from products.warehouse_sources.backend.models.table import (- HIDDEN_COLUMNS,- DataWarehouseTable as _DataWarehouseTable,-)+from products.warehouse_sources.backend.models.table import DataWarehouseTable as _DataWarehouseTable# Framework-free helper transforms — re-exported as the public helper surface.from products.warehouse_sources.backend.models.util import (@@ -610,6 +607,7 @@ def all_queryable_table_columns(team_id: int, table_ids: Collection[UUID] | None"id","name","columns",+ "options","external_data_source_id","external_data_source__id","external_data_source__prefix",@@ -628,10 +626,11 @@ def _hogql_visible_columns(table: _DataWarehouseTable) -> dict[str, str]:# The same raw -> visible mapping DataWarehouseTable.get_user_facing_columns applies, keeping# the stored type rather than the cleaned one so a consumer still reads any Nullable() wrapper.hogql_by_raw = get_hogql_column_name_mapping(table.table_name_without_prefix())+ hidden = table.hidden_columns()return {hogql_by_raw.get(name, name): type_for name, entry in (table.columns or {}).items()- if name not in HIDDEN_COLUMNS and (type_ := _clickhouse_column_type(entry)) is not None+ if name not in hidden and (type_ := _clickhouse_column_type(entry)) is not None}diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.pyindex f3ef8ef4d5e4..d4d5457ff0a3 100644--- a/products/warehouse_sources/backend/facade/source_management.py+++ b/products/warehouse_sources/backend/facade/source_management.py@@ -29,6 +29,8 @@ _LAZY = {"tables_wait_for_repair": "cdc.broken","purge_buffer_prefix": "cdc.buffer","CDC_SEQ_COLUMN": "cdc.batcher",+ "is_history_table_mode": "cdc.batcher",+ "history_column_collisions": "cdc.batcher","BUFFER_LANE": "cdc.snapshot_lane","CDC_RESET_PENDING_KEY": "cdc.snapshot_lane","hand_reset_to_capture_if_sync_running": "cdc.snapshot_lane",diff --git a/products/warehouse_sources/backend/models/table.py b/products/warehouse_sources/backend/models/table.pyindex 674a0218a469..5bc8bb2efdea 100644--- a/products/warehouse_sources/backend/models/table.py+++ b/products/warehouse_sources/backend/models/table.py@@ -187,6 +187,15 @@ type DataWarehouseTableIntrospectedColumns = dict[str, DataWarehouseTableIntrosp# and never user-facing.HIDDEN_COLUMNS: frozenset[str] = frozenset({"_dlt_id", "_dlt_load_id", "_ph_debug", PARTITION_KEY})+# Capture bookkeeping that a history table carries next to `valid_from`, `valid_to` and `_ph_deleted`,+# which already tell a reader when a version held and whether it is a delete. The names mirror the+# constants in `cdc/batcher.py`, which this module does not import because that pulls in pyarrow.+HISTORY_HIDDEN_COLUMNS: frozenset[str] = frozenset({"_ph_cdc_op", "_ph_cdc_seq", "_ph_cdc_timestamp", "_ph_deleted_at"})+# Set in `options` when the sync registers a table as a schema's history table. Column names cannot+# tell the two apart, because a source table can have its own `valid_from` and `valid_to` columns.+HISTORY_TABLE_OPTION = "history_table"++# Characters that carry quoting meaning once a column name is written into SQL. The structure# builder escapes them already, so this is a second layer: it keeps such a name out of the other# sinks that read this column store, not all of which escape as thoroughly (for example@@ -603,6 +612,19 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletprefix = ""return self.name[len(prefix) :]+ @property+ def is_history_table(self) -> bool:+ return (self.options or {}).get(HISTORY_TABLE_OPTION) is True++ def mark_as_history_table(self) -> None:+ self.options = {**(self.options or {}), HISTORY_TABLE_OPTION: True}++ def hidden_columns(self) -> frozenset[str]:+ """The plumbing columns to keep out of the catalog for this table."""+ if self.is_history_table:+ return HIDDEN_COLUMNS | HISTORY_HIDDEN_COLUMNS+ return HIDDEN_COLUMNS+def get_user_facing_columns(self) -> list[dict[str, Any]]:"""Synced columns as `[{name, data_type, is_nullable}]`, skipping internal plumbing columns.@@ -618,9 +640,10 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Delet`get_hogql_column_name_mapping(self.table_name_without_prefix())` directly."""hogql_by_raw = get_hogql_column_name_mapping(self.table_name_without_prefix())+ hidden = self.hidden_columns()result: list[dict[str, Any]] = []for name, definition in (self.columns or {}).items():- if name in HIDDEN_COLUMNS:+ if name in hidden:continueif isinstance(definition, dict):clickhouse_type = definition.get("clickhouse") or definition.get("hogql") or ""@@ -1298,6 +1321,11 @@ class DataWarehouseTable(CreatedMetaFields, UpdatedMetaFields, UUIDTModel, Deletif fields.get(PARTITION_KEY):del fields[PARTITION_KEY]fields = {**fields, **default_fields}+ # Hidden rather than removed, so a saved query that names one of these still resolves.+ if self.is_history_table:+ for name in HISTORY_HIDDEN_COLUMNS:+ if (field := fields.get(name)) is not None:+ fields[name] = field.model_copy(update={"hidden": True})table_def = HogQLDataWarehouseTable(name=self.name,diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.pyindex cdc02a3eaa96..55f1dc54fa8f 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py@@ -74,6 +74,8 @@ from products.warehouse_sources.backend.facade.source_management import (filter_dwh_columns_by_enabled_columns as _filter_dwh_columns_by_enabled_columns,get_cdc_adapter,hand_reset_to_capture_if_sync_running,+ history_column_collisions,+ is_history_table_mode,purge_buffer_prefix,repair_is_running,resnapshot_stays_in_buffer,@@ -444,20 +446,55 @@ def _apply_primary_key_columns()-def _refuse_reserved_cdc_column(instance: ExternalDataSchema) -> None:- """Capture stamps each change with this column, so a source column of the same name would fail- the source's sync. Checked against the columns discovery recorded, which are the source's own."""+def _source_column_names(instance: ExternalDataSchema) -> set[str]:+ """The columns discovery recorded, which are the source's own."""metadata = instance.schema_metadata or {}columns = metadata.get("columns") if isinstance(metadata, dict) else None- if isinstance(columns, list) and any(- isinstance(column, dict) and column.get("name") == CDC_SEQ_COLUMN for column in columns- ):+ if not isinstance(columns, list):+ return set()+ return {column["name"] for column in columns if isinstance(column, dict) and isinstance(column.get("name"), str)}+++def _refuse_reserved_cdc_column(instance: ExternalDataSchema) -> None:+ """Capture stamps each change with this column, so a source column of the same name would fail+ the source's sync."""+ if CDC_SEQ_COLUMN in _source_column_names(instance):raise ValidationError("Change data capture can't sync a column named _ph_cdc_seq, because PostHog uses that name. ""Rename the column on your database, or choose another sync method for this table.")+def _refuse_reserved_history_columns(+ instance: ExternalDataSchema,+ *,+ previous_mode: str | None,+ previous_enabled_columns: list[str] | None,+ previous_primary_keys: list[str] | None,+ new_mode: str | None,+ new_enabled_columns: list[str] | None,+ new_primary_keys: list[str] | None,+) -> None:+ """The history table stamps each row version with `valid_from` and `valid_to`, so a synced source+ column of the same name fails every load into it. Only a collision this request adds is refused,+ so a table that already has one can still be edited or switched back."""+ if not is_history_table_mode(new_mode):+ return+ columns = _source_column_names(instance)+ taken = history_column_collisions(columns, new_enabled_columns, new_primary_keys, instance.incremental_field)+ if instance.sync_type == ExternalDataSchema.SyncType.CDC and is_history_table_mode(previous_mode):+ existing = history_column_collisions(+ columns, previous_enabled_columns, previous_primary_keys, instance.incremental_field+ )+ taken = [column for column in taken if column not in existing]+ if taken:+ raise ValidationError(+ f"A history table can't store the column {' or '.join(taken)}, because PostHog uses valid_from and "+ "valid_to to record when each version was current. Rename the column on your database, or choose "+ "the consolidated table only for this table."+ )++def schema_display_status(schema: ExternalDataSchema) -> str | None:"""The user-facing sync status, mapping the two billing-limit statuses to friendly labels.Shared by the full and list schema serializers so the labels stay identical."""@@ -1318,6 +1355,20 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializerspayload["cdc_table_mode"] = cdc_table_modevalidated_data["sync_type_config"] = payload+ # Checked once the branches above settle the mode and the key, because `enabled_columns` alone+ # can also bring a validity column into the history table.+ if (sync_type if "sync_type" in data else instance.sync_type) == ExternalDataSchema.SyncType.CDC:+ resulting_config = validated_data.get("sync_type_config", instance.sync_type_config) or {}+ _refuse_reserved_history_columns(+ instance,+ previous_mode=previous_cdc_table_mode,+ previous_enabled_columns=instance.enabled_columns,+ previous_primary_keys=original_sync_type_config.get("primary_key_columns"),+ new_mode=resulting_config.get("cdc_table_mode"),+ new_enabled_columns=validated_data.get("enabled_columns", instance.enabled_columns),+ new_primary_keys=resulting_config.get("primary_key_columns"),+ )+should_sync = validated_data.get("should_sync", None)sync_frequency = data.get("sync_frequency", None)sync_time_of_day_in_payload = "sync_time_of_day" in datadiff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.pyindex 79630b5eef02..9313df03f593 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/source_setup.py@@ -69,6 +69,8 @@ from products.warehouse_sources.backend.facade.source_management import (fetch_docs_text,filter_dwh_columns_by_enabled_columns,get_cdc_adapter,+ history_column_collisions,+ is_history_table_mode,is_team_allowlisted_for_internal_hosts,new_source_requires_ssl,sql_schema_metadata,@@ -93,6 +95,14 @@ from products.warehouse_sources.backend.presentation.views.source_api_versions ifrom . import base, connection_options, credential_store, helpers, webhook_setup+def _payload_enabled_columns(schema: dict[str, Any]) -> list[str] | None:+ """`None` syncs every column. `[]` syncs only the always-retained key and incremental field."""+ payload_enabled_columns = schema.get("enabled_columns")+ if not isinstance(payload_enabled_columns, list):+ return None+ return [str(column) for column in payload_enabled_columns if isinstance(column, str)]++class ExternalDataSourceSerializers(UserAccessControlSerializerMixin, serializers.ModelSerializer):account_id = serializers.CharField(write_only=True)client_secret = serializers.CharField(write_only=True)@@ -1451,6 +1461,37 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):},)+ # The history table stamps each row version with these columns, so a synced source column+ # of the same name would fail every load into it.+ tables_with_validity_column = sorted(+ {+ schema["name"]+ for schema in payload_schemas+ if schema.get("sync_type") == "cdc"+ and schema.get("should_sync", False)+ and is_history_table_mode(schema.get("cdc_table_mode"))+ and isinstance(schema.get("name"), str)+ and history_column_collisions(+ [column[0] for column in getattr(source_schemas_by_name.get(schema["name"]), "columns", [])],+ _payload_enabled_columns(schema),+ pk_columns_by_table.get(schema["name"]),+ )+ }+ )+ if tables_with_validity_column:+ new_source_model.delete()+ return Response(+ status=status.HTTP_400_BAD_REQUEST,+ data={+ "message": (+ "A history table can't store a column that loads as valid_from or valid_to, because "+ "PostHog uses those names to record when each version was current. Rename the column on "+ "your database, or choose the consolidated table only for these tables: "+ f"{', '.join(tables_with_validity_column)}."+ )+ },+ )+# Engine-side CDC resource setup runs after PK validation so we don't leave# replication state on the source for a config we're about to refuse.if cdc_enabled:@@ -1476,15 +1517,7 @@ class ExternalDataSourceSetupMixin(base.ExternalDataSourceViewSetBase):primary_key_columns = schema.get("primary_key_columns")sync_time_of_day = schema.get("sync_time_of_day")should_sync = schema.get("should_sync", False)- payload_enabled_columns = schema.get("enabled_columns")- if isinstance(payload_enabled_columns, list):- # `[]` and `None` are distinct: `None` means sync all columns, `[]` means- # sync only the always-retained PK + incremental field.- enabled_columns: list[str] | None = [- str(column) for column in payload_enabled_columns if isinstance(column, str)- ]- else:- enabled_columns = None+ enabled_columns = _payload_enabled_columns(schema)payload_row_filters = schema.get("row_filters")row_filters: list[dict[str, Any]] | None = (diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.pyindex ce0c881c2486..278a583313ad 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/batcher.py@@ -1,12 +1,13 @@from __future__ import annotationsfrom collections import defaultdict-from collections.abc import Callable+from collections.abc import Callable, Iterableimport pyarrow as pafrom products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnErrorfrom products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent+from products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention# CDC metadata column names — database-agnosticCDC_OP_COLUMN = "_ph_cdc_op"@@ -47,6 +48,33 @@ TOAST_OMITTED_COLUMN = "_ph_toast_omitted"SCD2_VALID_FROM_COLUMN = "valid_from"SCD2_VALID_TO_COLUMN = "valid_to"+# Values of `cdc_table_mode` that write the history table.+CDC_HISTORY_TABLE_MODES: frozenset[str] = frozenset({"cdc_only", "both"})+++def is_history_table_mode(cdc_table_mode: object) -> bool:+ # The create payload is free-form, so a malformed mode can be stored, and a list or dict is unhashable.+ return isinstance(cdc_table_mode, str) and cdc_table_mode in CDC_HISTORY_TABLE_MODES+++def history_column_collisions(+ column_names: Iterable[str],+ enabled_columns: list[str] | None,+ primary_keys: list[str] | None,+ incremental_field: str | None = None,+) -> list[str]:+ """Source columns that the history table would receive under its validity column names."""+ synced = set(column_names)+ # The same projection capture applies (`filter_columns_by_enabled_columns`), kept inline because+ # that module's package imports Django models and this one sits on the capture path.+ if enabled_columns is not None:+ synced &= {*enabled_columns, *(primary_keys or []), *([incremental_field] if incremental_field else [])}+ # The load renames columns to snake_case before the history transform adds the validity columns,+ # so `validFrom` collides too. The source names are returned, because those are what a user renames.+ validity_columns = {SCD2_VALID_FROM_COLUMN, SCD2_VALID_TO_COLUMN}+ return sorted(name for name in synced if NamingConvention.normalize_identifier(name) in validity_columns)++# Columns that are CDC/SCD2 metadata — always filled from the event itself,# never overwritten when enriching DELETE rows with last-known source data._CDC_METADATA_COLUMNS: frozenset[str] = frozenset(diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.pyindex 4d05f38c9086..ce94cc745109 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_batcher.py@@ -22,6 +22,8 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher importbuild_scd2_table,enrich_delete_rows,enrich_toast_omitted_rows,+ history_column_collisions,+ is_history_table_mode,)from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCReservedColumnErrorfrom products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent@@ -711,6 +713,41 @@ class TestScd2TimestampType:assert result.column(SCD2_VALID_TO_COLUMN).to_pylist()[0] is not None+class TestHistoryColumnCollisions:+ @parameterized.expand(+ [+ ("exact_name", ["id", "valid_to"], None, None, ["valid_to"]),+ ("name_the_load_renames", ["id", "validFrom", "VALID_TO"], None, None, ["VALID_TO", "validFrom"]),+ ("left_out_of_enabled_columns", ["id", "validFrom"], ["id"], None, []),+ ("kept_as_the_incremental_field", ["id", "valid_from"], [], "valid_from", ["valid_from"]),+ ]+ )+ def test_names_the_source_columns_that_collide(+ self,+ _name: str,+ column_names: list[str],+ enabled_columns: list[str] | None,+ incremental_field: str | None,+ expected: list[str],+ ) -> None:+ assert history_column_collisions(column_names, enabled_columns, ["id"], incremental_field) == expected+++class TestIsHistoryTableMode:+ @parameterized.expand(+ [+ ("cdc_only", "cdc_only", True),+ ("both", "both", True),+ ("consolidated", "consolidated", False),+ ("missing", None, False),+ ("list", [], False),+ ("dict", {}, False),+ ]+ )+ def test_reads_only_a_history_mode_string(self, _name: str, cdc_table_mode: object, expected: bool) -> None:+ assert is_history_table_mode(cdc_table_mode) is expected++class TestScd2ReservedColumns:@parameterized.expand([(SCD2_VALID_FROM_COLUMN,), (SCD2_VALID_TO_COLUMN,)])def test_a_source_column_named_like_the_stamp_fails_before_the_writer(self, taken):diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.pyindex 2c71e846636d..edb2a2906a3e 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_sync.py@@ -34,7 +34,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (update_sync_type_config_keys,)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource-from products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTablefrom products.warehouse_sources.backend.models.util import hogql_type_name_for_clickhouse_typefrom products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import companion_resource_namefrom products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConvention@@ -699,6 +699,7 @@ async def register_cdc_companion_table(table.format = table_formattable.url_pattern = new_url_patterntable.queryable_folder = queryable_folder+ table.mark_as_history_table()_refresh_cumulative_row_count(table, logger, companion_table_name, live_row_count)# Scope to the fields changed here so this out-of-transaction save doesn't rewrite# `columns` with its pre-merge value before the column save below.@@ -707,7 +708,7 @@ async def register_cdc_companion_table(# on a fresh connection rather than let this escape as error-tracking noise.# new_url_pattern above is derived from the job's own destination folder, not from# request input, so this sync is a trusted writer of a credential-less table's URL.- update_fields = ["format", "url_pattern", "queryable_folder", "row_count"]+ update_fields = ["format", "url_pattern", "queryable_folder", "options", "row_count"]if live_size_mib is not None:table.size_in_s3_mib = live_size_mibupdate_fields.append("size_in_s3_mib")@@ -720,6 +721,7 @@ async def register_cdc_companion_table(external_data_source_id=job.pipeline.id,created_via=DataWarehouseTableCreatedVia.SOURCE,size_in_s3_mib=live_size_mib,+ options={HISTORY_TABLE_OPTION: True},**table_params,)_record_storage_delta(job, storage_delta_mib)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.pyindex 64ade129ac45..3439a609ac59 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/test_pipeline_sync.py@@ -22,7 +22,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (SelfManagedTableRef,)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource-from products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.models.table import HISTORY_TABLE_OPTION, DataWarehouseTablefrom products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConventionfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.helpers import (build_table_name,@@ -406,6 +406,8 @@ class TestRegisterCdcCompanionTableStamp:job.refresh_from_db()assert companion.size_in_s3_mib == 5.0assert job.storage_delta_mib == 5.0+ # The catalog reads this marker to hide the capture columns.+ assert companion.options[HISTORY_TABLE_OPTION] is True# transaction=True: validate_schema_and_update_table writes to the DB from the async thread pooldiff --git a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.pyindex aaf7e0f6f953..363be89fa294 100644--- a/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py+++ b/products/warehouse_sources/backend/temporal/data_imports/workflow_activities/create_job_model.py@@ -25,7 +25,7 @@ from products.warehouse_sources.backend.models.external_data_schema import (mark_schema_running_unless_halted,)from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource-from products.warehouse_sources.backend.models.table import HIDDEN_COLUMNS, DataWarehouseTable+from products.warehouse_sources.backend.models.table import DataWarehouseTablefrom products.warehouse_sources.backend.temporal.data_imports.destinations.enablement import (NoActiveDestinationsError,destination_ids_for_run,@@ -141,7 +141,7 @@ def _enrichment_pending(team_id: int, table: DataWarehouseTable | None, schema:return True# Hidden plumbing columns (_dlt_id, partition key, …) are never enriched, so they'd otherwise# show up perpetually in `columns - annotated` and keep this returning True on every sync.- columns = set((table.columns or {}).keys()) - HIDDEN_COLUMNS+ columns = set((table.columns or {}).keys()) - table.hidden_columns()annotated = set(WarehouseColumnAnnotation.objects.for_team(team_id).filter(table_id=table.id)diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py b/products/warehouse_sources/backend/tests/api/test_external_data_schema.pyindex 6f2da38fdba0..f70f4e4deb07 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_schema.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_schema.py@@ -17,6 +17,7 @@ import pytest_asynciofrom asgiref.sync import sync_to_asyncfrom parameterized import parameterizedfrom rest_framework import status+from rest_framework.exceptions import ValidationErrorfrom temporalio.service import RPCError, RPCStatusCodefrom posthog.api.test.test_organization import create_organization@@ -1340,6 +1341,30 @@ class TestExternalDataSchema(APIBaseTest):None,"_ph_cdc_seq",),+ (+ "rejects_a_history_table_for_a_source_column_named_like_a_validity_column",+ {+ "primary_key_columns": ["id"],+ "cdc_table_mode": "both",+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "valid_from"}]},+ },+ None,+ status.HTTP_400_BAD_REQUEST,+ ExternalDataSchema.SyncType.FULL_REFRESH,+ None,+ "valid_from",+ ),+ (+ "accepts_a_validity_column_name_without_a_history_table",+ {+ "primary_key_columns": ["id"],+ "schema_metadata": {"columns": [{"name": "id"}, {"name": "valid_from"}]},+ },+ None,+ status.HTTP_200_OK,+ ExternalDataSchema.SyncType.CDC,+ ["id"],+ ),])def test_update_schema_to_cdc(@@ -4503,6 +4528,74 @@ class TestSerializerLostUpdateProtection(APIBaseTest):assert self.schema.cdc_table_mode == "both"assert self.schema.sync_type_config["cdc_last_log_position"] == "0/900"+ @parameterized.expand(+ [+ (+ "turning_history_on_over_a_synced_column_is_refused",+ "consolidated",+ None,+ {"cdc_table_mode": "both"},+ True,+ ),+ (+ "turning_history_on_over_a_column_left_out_is_allowed",+ "consolidated",+ ["id"],+ {"cdc_table_mode": "both"},+ False,+ ),+ (+ "adding_the_column_to_enabled_columns_is_refused",+ "both",+ ["id"],+ {"enabled_columns": ["id", "valid_to"]},+ True,+ ),+ ("editing_a_table_that_already_syncs_it_is_allowed", "both", None, {"enabled_columns": None}, False),+ ]+ )+ def test_patch_checks_the_validity_columns_the_history_table_receives(+ self,+ _name: str,+ initial_mode: str,+ initial_enabled_columns: list[str] | None,+ request_body: dict,+ refused: bool,+ ):+ from products.warehouse_sources.backend.presentation.views.external_data_schema import (+ ExternalDataSchemaSerializer,+ )++ self.schema.sync_type_config["cdc_table_mode"] = initial_mode+ self.schema.sync_type_config["schema_metadata"] = {"columns": [{"name": "id"}, {"name": "valid_to"}]}+ self.schema.enabled_columns = initial_enabled_columns+ self.schema.save()++ serializer = ExternalDataSchemaSerializer(+ self.schema,+ data=request_body,+ partial=True,+ context={"team_id": self.team.pk, "post_commit_actions": []},+ )+ with mock.patch(+ "products.warehouse_sources.backend.presentation.views.external_data_schema.is_any_external_data_schema_paused",+ return_value=False,+ ):+ serializer.is_valid(raise_exception=True)+ if refused:+ with pytest.raises(ValidationError, match="valid_to"):+ serializer.save()+ else:+ serializer.save()++ self.schema.refresh_from_db()+ assert self.schema.cdc_table_mode == (+ initial_mode if refused else request_body.get("cdc_table_mode", initial_mode)+ )+ assert self.schema.enabled_columns == (+ initial_enabled_columns if refused else request_body.get("enabled_columns", initial_enabled_columns)+ )+def test_patch_does_not_revert_a_concurrent_delete_data(self):from products.warehouse_sources.backend.presentation.views.external_data_schema import (ExternalDataSchemaSerializer,diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex f413e9ce6327..57fd7be8869e 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -4974,12 +4974,30 @@ class TestExternalDataSource(APIBaseTest):@parameterized.expand([- ("no_primary_key", [("id", "uuid", False)], {}, "primary key"),+ ("no_primary_key", [("id", "uuid", False)], {}, "primary key", "consolidated", None),("reserved_column",[("id", "uuid", False), ("_ph_cdc_seq", "bigint", True)],{"tracking_link": ["id"]},"_ph_cdc_seq",+ "consolidated",+ None,+ ),+ (+ "validity_column_with_a_history_table",+ [("id", "uuid", False), ("valid_to", "timestamp", True)],+ {"tracking_link": ["id"]},+ "valid_to",+ "cdc_only",+ None,+ ),+ (+ "validity_column_in_enabled_columns_that_hold_a_non_string",+ [("id", "uuid", False), ("valid_to", "timestamp", True)],+ {"tracking_link": ["id"]},+ "valid_to",+ "cdc_only",+ ["id", "valid_to", {}],),])@@ -5002,6 +5020,8 @@ class TestExternalDataSource(APIBaseTest):columns,primary_keys,expected_in_message,+ cdc_table_mode,+ enabled_columns,mock_get_source,mock_cdc_pg_connection,mock_get_primary_key_columns,@@ -5059,7 +5079,13 @@ class TestExternalDataSource(APIBaseTest):"schema": "public","cdc_enabled": True,"schemas": [- {"name": "tracking_link", "should_sync": True, "sync_type": "cdc"},+ {+ "name": "tracking_link",+ "should_sync": True,+ "sync_type": "cdc",+ "cdc_table_mode": cdc_table_mode,+ **({"enabled_columns": enabled_columns} if enabled_columns is not None else {}),+ },],},},diff --git a/products/warehouse_sources/backend/tests/test_table.py b/products/warehouse_sources/backend/tests/test_table.pyindex df8b293fb814..3ea501642881 100644--- a/products/warehouse_sources/backend/tests/test_table.py+++ b/products/warehouse_sources/backend/tests/test_table.py@@ -33,11 +33,18 @@ from posthog.models import Teamfrom products.warehouse_sources.backend.models.credential import DataWarehouseCredentialfrom products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom products.warehouse_sources.backend.models.table import (+ HISTORY_HIDDEN_COLUMNS,DataWarehouseTable,classify_warehouse_read_error,get_hogql_field_for_column,run_chdb_query,)+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (+ CDC_OP_COLUMN,+ CDC_SEQ_COLUMN,+ CDC_TIMESTAMP_COLUMN,+ DELETED_AT_COLUMN,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.errors import (TransientObjectStoreError,)@@ -719,6 +726,11 @@ class TestSchemaInferenceMode(BaseTest):assert mock_sync_execute.call_count == 5+class TestHistoryHiddenColumns(SimpleTestCase):+ def test_names_match_the_columns_capture_writes(self) -> None:+ assert HISTORY_HIDDEN_COLUMNS == {CDC_OP_COLUMN, CDC_SEQ_COLUMN, CDC_TIMESTAMP_COLUMN, DELETED_AT_COLUMN}++class TestGetHogqlFieldForColumn(SimpleTestCase):@parameterized.expand([