Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): show the real history table name and backfill the marker (#125676)

Daniel RCbe9f7bc96c78
diff --git a/frontend/src/types.ts b/frontend/src/types.ts
index 250ba99a03aa..de722dbafddf 100644
--- a/frontend/src/types.ts
+++ b/frontend/src/types.ts
@@ -6715,6 +6715,8 @@ export interface ExternalDataSourceSchema extends SimpleExternalDataSourceSchema
should_sync_default?: boolean
primary_key_columns: string[] | null
cdc_table_mode?: 'consolidated' | 'cdc_only' | 'both'
+ /** The history table's name, or `null` when the table keeps no history. Older history tables end in `_cdc`. */
+ history_table_name?: string | null
/**
* Why the last sync run could not merge rows on this table's primary key, or `null` when no such
* failure is current. A later run that succeeds, or fails for another reason, clears it.
diff --git a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
index 5bd874f9f05b..b94091d25e7f 100644
--- a/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SchemaScene/ConfigurationTab.tsx
@@ -1323,8 +1323,6 @@ function DangerZoneSection({
type="secondary"
status="danger"
onClick={() => {
- const hasCdcTable =
- schema.cdc_table_mode === 'cdc_only' || schema.cdc_table_mode === 'both'
LemonDialog.open({
title: 'Full resync — all existing data will be replaced',
content: (
@@ -1335,15 +1333,12 @@ function DangerZoneSection({
<strong>{schema.table?.name ?? schema.name}</strong> table
will be replaced with the new snapshot.
</p>
- {hasCdcTable && (
+ {schema.history_table_name && (
<p>
- The{' '}
- <strong>
- {(schema.table?.name ?? schema.name) + '_cdc'}
- </strong>{' '}
- history table will also be reset — all change history
- will be lost and replaced with the new snapshot as the
- starting point.
+ The <strong>{schema.history_table_name}</strong> history
+ table will also be reset — all change history will be
+ lost and replaced with the new snapshot as the starting
+ point.
</p>
)}
</div>
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
index 644dd1257aae..ff378ea5b529 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/SchemasTab.tsx
@@ -1125,8 +1125,6 @@ function SchemaRowMore({
fullWidth
status="danger"
onClick={() => {
- const hasCdcTable =
- schema.cdc_table_mode === 'cdc_only' || schema.cdc_table_mode === 'both'
LemonDialog.open({
title: 'Full resync — all existing data will be replaced',
content: (
@@ -1137,15 +1135,12 @@ function SchemaRowMore({
<strong>{schema.table?.name ?? schema.name}</strong> table
will be replaced with the new snapshot.
</p>
- {hasCdcTable && (
+ {schema.history_table_name && (
<p>
- The{' '}
- <strong>
- {(schema.table?.name ?? schema.name) + '_cdc'}
- </strong>{' '}
- history table will also be reset — all change history
- will be lost and replaced with the new snapshot as the
- starting point.
+ The <strong>{schema.history_table_name}</strong> history
+ table will also be reset — all change history will be
+ lost and replaced with the new snapshot as the starting
+ point.
</p>
)}
</div>
diff --git a/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsx b/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsx
index 383ef0ccca32..ac948c46b4bd 100644
--- a/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsx
+++ b/products/data_warehouse/frontend/shared/components/forms/SyncMethodForm.tsx
@@ -336,8 +336,8 @@ export const SyncMethodForm = forwardRef<SyncMethodFormHandle, SyncMethodFormPro
<div className="font-normal mb-2">
<div className="font-semibold">CDC history table only</div>
<p className="m-0 text-secondary text-sm">
- Full audit trail in a <code>_cdc</code>-suffixed table with{' '}
- <code>valid_from</code> / <code>valid_to</code> columns.
+ Full audit trail in a separate history table, with a validity
+ interval on every row.
</p>
</div>
),
@@ -348,8 +348,7 @@ export const SyncMethodForm = forwardRef<SyncMethodFormHandle, SyncMethodFormPro
<div className="font-normal mb-2">
<div className="font-semibold">Both</div>
<p className="m-0 text-secondary text-sm">
- Consolidated current-state table plus a separate <code>_cdc</code>
- -suffixed history table.
+ Consolidated current-state table plus a separate history table.
</p>
</div>
),
diff --git a/products/warehouse_sources/backend/management/commands/backfill_history_table_marker.py b/products/warehouse_sources/backend/management/commands/backfill_history_table_marker.py
new file mode 100644
index 000000000000..e63a3f4018d7
--- /dev/null
+++ b/products/warehouse_sources/backend/management/commands/backfill_history_table_marker.py
@@ -0,0 +1,52 @@
+"""Mark the history tables that never synced after the `history_table` marker existed.
+
+The sync marks a history table each time it writes to it. A table that stopped syncing earlier, such as
+one whose schema dropped its history table, stays unmarked, so the catalog shows its capture columns and
+the marker-based checks do not see it.
+"""
+
+from typing import Any
+
+from django.core.management.base import BaseCommand, CommandParser
+
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.models.table import DataWarehouseTable
+from products.warehouse_sources.backend.temporal.data_imports.cdc.history_table import history_tables
+
+
+class Command(BaseCommand):
+ help = (
+ "Set the history_table marker on registered legacy `_cdc` history tables that lack it. "
+ "Previews by default; pass --live-run to write."
+ )
+
+ def add_arguments(self, parser: CommandParser) -> None:
+ parser.add_argument("--team-id", type=int, default=None, help="Only mark tables of this team.")
+ parser.add_argument(
+ "--live-run", action="store_true", help="Write the marker. Without this the command only reports."
+ )
+
+ def handle(self, *args: Any, **options: Any) -> None:
+ # Only a legacy history table can be unmarked, and every one ends in `_cdc`, so this narrows the
+ # sources before each one is resolved exactly.
+ candidates = DataWarehouseTable.objects.filter(
+ deleted=False, external_data_source__isnull=False, name__endswith="_cdc"
+ )
+ if options["team_id"] is not None:
+ candidates = candidates.filter(team_id=options["team_id"])
+ sources = ExternalDataSource.objects.filter(id__in=candidates.values("external_data_source_id"))
+
+ marked = 0
+ scanned = 0
+ # Streamed, so memory stays flat across every source of the installation.
+ for source in sources.iterator(chunk_size=500):
+ scanned += 1
+ for table in (table for table in history_tables(source) if not table.is_history_table):
+ self.stdout.write(f"team={table.team_id} source={source.id} table={table.id} name={table.name}")
+ marked += 1
+ if options["live_run"]:
+ table.mark_as_history_table()
+ table.save(update_fields=["options"])
+
+ verb = "Marked" if options["live_run"] else "Would mark"
+ self.stdout.write(f"{verb} {marked} table(s) across {scanned} candidate source(s).")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py
index c24b8ee21b43..0d9e3029a3bf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_history_table.py
@@ -1,9 +1,12 @@
import uuid
+from io import StringIO
import pytest
from posthog.test.base import BaseTest
from unittest.mock import patch
+from django.core.management import call_command
+
from parameterized import parameterized
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
@@ -190,3 +193,17 @@ class TestHistoryNaming(BaseTest):
)
assert set(history_tables(self.source)) == {legacy, marked}
+
+ @parameterized.expand([("preview", False), ("live_run", True)])
+ def test_backfill_marks_unmarked_legacy_history_tables(self, _name: str, live_run: bool) -> None:
+ legacy = self._table("orders_cdc")
+ ExternalDataSchema.objects.create(name="items", team=self.team, source=self.source)
+ own_table = self._table("items_cdc")
+ ExternalDataSchema.objects.create(name="items_cdc", team=self.team, source=self.source, table=own_table)
+
+ call_command("backfill_history_table_marker", *(["--live-run"] if live_run else []), stdout=StringIO())
+
+ legacy.refresh_from_db()
+ own_table.refresh_from_db()
+ assert legacy.is_history_table is live_run
+ assert not own_table.is_history_table