:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): show the real history table name and backfill the marker (#125676)
Daniel RCbe9f7bc96c78
diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex 250ba99a03aa..de722dbafddf 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -6715,6 +6715,8 @@ export interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchemashould_sync_default?: booleanprimary_key_columns: string[] | nullcdc_table_mode?: 'consolidated' | 'cdc_only' | 'both'+ /** The history table's name, or `null` when the table keeps no history. Older history tables end in `_cdc`. */+ history_table_name?: string | null/*** Why the last sync run could not merge rows on this table's primary key, or `null` when no such* failure is current. A later run that succeeds, or fails for another reason, clears it.diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsxindex 5bd874f9f05b..b94091d25e7f 100644--- a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx+++ b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx@@ -1323,8 +1323,6 @@ function DangerZoneSection({type="secondary"status="danger"onClick={() => {- const hasCdcTable =- schema.cdc_table_mode === 'cdc_only' || schema.cdc_table_mode === 'both'LemonDialog.open({title: 'Full resync — all existing data will be replaced',content: (@@ -1335,15 +1333,12 @@ function DangerZoneSection({<strong>{schema.table?.name ?? schema.name}</strong> tablewill be replaced with the new snapshot.</p>- {hasCdcTable && (+ {schema.history_table_name && (<p>- The{' '}- <strong>- {(schema.table?.name ?? schema.name) + '_cdc'}- </strong>{' '}- history table will also be reset — all change history- will be lost and replaced with the new snapshot as the- starting point.+ The <strong>{schema.history_table_name}</strong> history+ table will also be reset — all change history will be+ lost and replaced with the new snapshot as the starting+ point.</p>)}</div>diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsxindex 644dd1257aae..ff378ea5b529 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx@@ -1125,8 +1125,6 @@ function SchemaRowMore({fullWidthstatus="danger"onClick={() => {- const hasCdcTable =- schema.cdc_table_mode === 'cdc_only' || schema.cdc_table_mode === 'both'LemonDialog.open({title: 'Full resync — all existing data will be replaced',content: (@@ -1137,15 +1135,12 @@ function SchemaRowMore({<strong>{schema.table?.name ?? schema.name}</strong> tablewill be replaced with the new snapshot.</p>- {hasCdcTable && (+ {schema.history_table_name && (<p>- The{' '}- <strong>- {(schema.table?.name ?? schema.name) + '_cdc'}- </strong>{' '}- history table will also be reset — all change history- will be lost and replaced with the new snapshot as the- starting point.+ The <strong>{schema.history_table_name}</strong> history+ table will also be reset — all change history will be+ lost and replaced with the new snapshot as the starting+ point.</p>)}</div>diff --git a/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsx b/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsxindex 383ef0ccca32..ac948c46b4bd 100644--- a/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsx+++ b/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsx@@ -336,8 +336,8 @@ export const SyncMethodForm = forwardRef<SyncMethodFormHandle, SyncMethodFormPro<div className="font-normal mb-2"><div className="font-semibold">CDC history table only</div><p className="m-0 text-secondary text-sm">- Full audit trail in a <code>_cdc</code>-suffixed table with{' '}- <code>valid_from</code> / <code>valid_to</code> columns.+ Full audit trail in a separate history table, with a validity+ interval on every row.</p></div>),@@ -348,8 +348,7 @@ export const SyncMethodForm = forwardRef<SyncMethodFormHandle, SyncMethodFormPro<div className="font-normal mb-2"><div className="font-semibold">Both</div><p className="m-0 text-secondary text-sm">- Consolidated current-state table plus a separate <code>_cdc</code>- -suffixed history table.+ Consolidated current-state table plus a separate history table.</p></div>),diff --git a/products/warehouse_sources/backend/management/commands/backfill_history_table_marker.py b/products/warehouse_sources/backend/management/commands/backfill_history_table_marker.pynew file mode 100644index 000000000000..e63a3f4018d7--- /dev/null+++ b/products/warehouse_sources/backend/management/commands/backfill_history_table_marker.py@@ -0,0 +1,52 @@+"""Mark the history tables that never synced after the `history_table` marker existed.++The sync marks a history table each time it writes to it. A table that stopped syncing earlier, such as+one whose schema dropped its history table, stays unmarked, so the catalog shows its capture columns and+the marker-based checks do not see it.+"""++from typing import Any++from django.core.management.base import BaseCommand, CommandParser++from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.models.table import DataWarehouseTable+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import history_tables+++class Command(BaseCommand):+ help = (+ "Set the history_table marker on registered legacy `_cdc` history tables that lack it. "+ "Previews by default; pass --live-run to write."+ )++ def add_arguments(self, parser: CommandParser) -> None:+ parser.add_argument("--team-id", type=int, default=None, help="Only mark tables of this team.")+ parser.add_argument(+ "--live-run", action="store_true", help="Write the marker. Without this the command only reports."+ )++ def handle(self, *args: Any, **options: Any) -> None:+ # Only a legacy history table can be unmarked, and every one ends in `_cdc`, so this narrows the+ # sources before each one is resolved exactly.+ candidates = DataWarehouseTable.objects.filter(+ deleted=False, external_data_source__isnull=False, name__endswith="_cdc"+ )+ if options["team_id"] is not None:+ candidates = candidates.filter(team_id=options["team_id"])+ sources = ExternalDataSource.objects.filter(id__in=candidates.values("external_data_source_id"))++ marked = 0+ scanned = 0+ # Streamed, so memory stays flat across every source of the installation.+ for source in sources.iterator(chunk_size=500):+ scanned += 1+ for table in (table for table in history_tables(source) if not table.is_history_table):+ self.stdout.write(f"team={table.team_id} source={source.id} table={table.id} name={table.name}")+ marked += 1+ if options["live_run"]:+ table.mark_as_history_table()+ table.save(update_fields=["options"])++ verb = "Marked" if options["live_run"] else "Would mark"+ self.stdout.write(f"{verb} {marked} table(s) across {scanned} candidate source(s).")diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.pyindex c24b8ee21b43..0d9e3029a3bf 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py@@ -1,9 +1,12 @@import uuid+from io import StringIOimport pytestfrom posthog.test.base import BaseTestfrom unittest.mock import patch+from django.core.management import call_command+from parameterized import parameterizedfrom products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema@@ -190,3 +193,17 @@ class TestHistoryNaming(BaseTest):)assert set(history_tables(self.source)) == {legacy, marked}++ @parameterized.expand([("preview", False), ("live_run", True)])+ def test_backfill_marks_unmarked_legacy_history_tables(self, _name: str, live_run: bool) -> None:+ legacy = self._table("orders_cdc")+ ExternalDataSchema.objects.create(name="items", team=self.team, source=self.source)+ own_table = self._table("items_cdc")+ ExternalDataSchema.objects.create(name="items_cdc", team=self.team, source=self.source, table=own_table)++ call_command("backfill_history_table_marker", *(["--live-run"] if live_run else []), stdout=StringIO())++ legacy.refresh_from_db()+ own_table.refresh_from_db()+ assert legacy.is_history_table is live_run+ assert not own_table.is_history_table