:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(data-warehouse): drive CDC setup and status gating from the adapter (#123574)
Daniel RC3f2b0a907112
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsxindex 34e7048e7302..20a63917e46a 100644--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsx+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsx@@ -32,7 +32,6 @@ import {externalDataSourcesResumeCdcCreate,} from 'products/warehouse_sources/frontend/generated/api'-import { CDC_SOURCE_TYPES } from '../../../shared/cdc'import { sourceSettingsLogic } from './sourceSettingsLogic'type ManagementMode = 'posthog' | 'self_managed'@@ -151,8 +150,9 @@ function confirmThen(opts: {export function CDCSection({ source }: { source: ExternalDataSource }): JSX.Element | null {const { featureFlags } = useValues(featureFlagLogic)+ const { sourceFieldConfig } = useValues(sourceSettingsLogic)- if (!CDC_SOURCE_TYPES.includes(source.source_type)) {+ if (!sourceFieldConfig?.cdcEngine) {return null}if (source.access_method !== 'warehouse') {diff --git a/products/data_warehouse/frontend/shared/cdc.ts b/products/data_warehouse/frontend/shared/cdc.tsdeleted file mode 100644index c136596c003e..000000000000--- a/products/data_warehouse/frontend/shared/cdc.ts+++ /dev/null@@ -1,4 +0,0 @@-// Source types that support change data capture (logical replication). Supabase and Neon-// are Postgres on the wire and reuse the same CDC path. Keep in sync with the backend-// adapter registry in products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py.-export const CDC_SOURCE_TYPES: string[] = ['Postgres', 'Supabase', 'Neon']diff --git a/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsx b/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsxindex 5ee692408d97..eb2cbabe3b4d 100644--- a/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsx+++ b/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsx@@ -35,7 +35,6 @@ import {type SourceWizardLogicProps,sourceWizardLogic,} from '../../../scenes/NewSourceScene/sourceWizardLogic'-import { CDC_SOURCE_TYPES } from '../../cdc'import { CredentialAccountSelector } from './CredentialAccountSelector'import { isCustomSourceAiBuilderEnabled } from './customSourceManifest'import { CustomSourceManifestBuilder } from './CustomSourceManifestBuilder'@@ -1030,7 +1029,7 @@ export function SourceFormComponent({</Group>{!isUpdateMode &&showCdcConfig &&- CDC_SOURCE_TYPES.includes(sourceConfig.name) &&+ !!sourceConfig.cdcEngine &&featureFlags[FEATURE_FLAGS.DWH_POSTGRES_CDC] &&selectedAccessMethod === 'warehouse' && <CDCConfigSection sourceName={sourceConfig.name} />}{showPrefix && !isDirectQuerySource && !customAiIntroActive && (diff --git a/products/warehouse_sources/backend/facade/source_config.py b/products/warehouse_sources/backend/facade/source_config.pyindex 70d4825418b2..a14dc182be6a 100644--- a/products/warehouse_sources/backend/facade/source_config.py+++ b/products/warehouse_sources/backend/facade/source_config.py@@ -14,7 +14,7 @@ from typing import Annotated, Any, Literal, Unionfrom pydantic import BaseModel, ConfigDict, Field, RootModelfrom pydantic.json_schema import SkipJsonSchema, WithJsonSchema-from products.warehouse_sources.backend.facade.types import ExternalDataSourceType+from products.warehouse_sources.backend.facade.types import CDCEngine, ExternalDataSourceType__all__ = ["DataWarehouseSourceCategory",@@ -442,6 +442,13 @@ class SourceConfigResponse(SourceConfig):description="Vendor API docs or changelog URL, or null when the vendor publishes none.",)deprecatedVersions: list[SourceVersionDeprecation]+ cdcEngine: CDCEngine | None = Field(+ default=None,+ description=(+ "Change data capture engine, which picks the CDC setup form and status panel. Null when"+ " the source has no CDC."+ ),+ )tables: list[SourceDocumentedTable] | None = Field(default=None,description=(diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.pyindex d4d5457ff0a3..3e36a91f6b1d 100644--- a/products/warehouse_sources/backend/facade/source_management.py+++ b/products/warehouse_sources/backend/facade/source_management.py@@ -19,6 +19,7 @@ _LAZY = {"fetch_docs_text": "sources.custom.ai_builder","CDCSourceAdapter": "cdc.adapters","add_table_failure_message": "cdc.adapters",+ "cdc_engine": "cdc.adapters","get_cdc_adapter": "cdc.adapters","source_type_supports_cdc": "cdc.adapters","CDCRepairError": "cdc.repair",diff --git a/products/warehouse_sources/backend/facade/types.py b/products/warehouse_sources/backend/facade/types.pyindex 2faa7d7fcf26..03e4c7be30d8 100644--- a/products/warehouse_sources/backend/facade/types.py+++ b/products/warehouse_sources/backend/facade/types.py@@ -1508,6 +1508,12 @@ DIRECT_ENGINE_BY_SOURCE_TYPE: dict[str, str] = {}+class CDCEngine(StrEnum):+ """Change data capture engine of a source type, which picks its CDC setup form and status panel."""++ POSTGRES = "postgres"++class ManagedWarehouseSQLMode(StrEnum):EXTERNAL = "external"BUILT_IN = "built_in"diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.pyindex 3b11bf4c09ea..677dc8d9d7d5 100644--- a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py@@ -720,7 +720,7 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):data={"message": f"Could not connect to source to resume CDC: {e}"},)- if live_status.get("slot_exists") is False or live_status.get("publication_exists") is False:+ if not adapter.capture_resources_present(live_status):return Response(status=status.HTTP_400_BAD_REQUEST,data={"message": "The replication slot or publication is missing. Use Repair CDC to recreate it."},@@ -860,8 +860,6 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):{"enabled": True,"management_mode": cdc_config.management_mode,- "slot_name": cdc_config.slot_name,- "publication_name": cdc_config.publication_name,"lag_warning_threshold_mb": cdc_config.lag_warning_threshold_mb,"lag_critical_threshold_mb": cdc_config.lag_critical_threshold_mb,"schedule_paused": schedule_paused,diff --git a/products/warehouse_sources/backend/presentation/views/public_source_configs.py b/products/warehouse_sources/backend/presentation/views/public_source_configs.pyindex d93683a9b277..b3805042aa2b 100644--- a/products/warehouse_sources/backend/presentation/views/public_source_configs.py+++ b/products/warehouse_sources/backend/presentation/views/public_source_configs.py@@ -8,7 +8,7 @@ from rest_framework.request import Requestfrom rest_framework.response import Responsefrom products.warehouse_sources.backend.facade.source_config import SourceConfigMapResponse-from products.warehouse_sources.backend.facade.source_management import SourceRegistry+from products.warehouse_sources.backend.facade.source_management import SourceRegistry, cdc_enginelogger = structlog.get_logger(__name__)tracer = trace.get_tracer(__name__)@@ -42,6 +42,7 @@ def build_source_configs(*, include_tables: bool = True) -> dict[str, dict]:config["versions"] = list(source.supported_versions)config["defaultVersion"] = source.default_versionconfig["apiDocsUrl"] = source.api_docs_url+ config["cdcEngine"] = cdc_engine(source_type)config["deprecatedVersions"] = [{"version": d.version, "sunsetAt": d.sunset_at.isoformat() if d.sunset_at else None}for d in source.deprecated_versions@@ -78,7 +79,8 @@ class PublicSourceConfigViewSet(viewsets.ViewSet):"documented table catalog; empty for SQL/file sources with user-defined schemas), and ""vendor API version metadata: `versions` (supported version labels), `defaultVersion`, ""`apiDocsUrl` (vendor API docs/changelog URL or null), and `deprecatedVersions` "- "(array of `{version, sunsetAt}` with `sunsetAt` an ISO date or null)."+ "(array of `{version, sunsetAt}` with `sunsetAt` an ISO date or null), and `cdcEngine` "+ "(the change data capture engine, or null when the source has no CDC)."),)def list(self, request: Request) -> Response:diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.pyindex 0e7a895cc24d..272e00d5e3e9 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py@@ -14,7 +14,7 @@ register it in ``_cdc_adapters`` below.from __future__ import annotations-from collections.abc import Iterator+from collections.abc import Iterator, Mappingfrom contextlib import contextmanagerfrom typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar@@ -22,7 +22,7 @@ from posthog.exceptions_capture import capture_exceptionfrom products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfigfrom products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapter-from products.warehouse_sources.backend.types import ExternalDataSourceType+from products.warehouse_sources.backend.types import CDCEngine, ExternalDataSourceTypeif TYPE_CHECKING:from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource@@ -40,6 +40,9 @@ class CDCSourceAdapter(Protocol[CDCConfigT_co]):is typed precisely without forcing callers to do ``isinstance`` checks."""+ # The catalog serves it as `cdcEngine`. Engines that are the same database on the wire share one.+ engine: CDCEngine+def create_reader(self, source: ExternalDataSource) -> CDCStreamReader: ...def position_to_seq(self, position_serialized: str) -> int:@@ -142,10 +145,17 @@ class CDCSourceAdapter(Protocol[CDCConfigT_co]):def get_status(self, source: ExternalDataSource) -> dict[str, Any]:"""Live engine-side CDC health for the source, read from the source DB.- Opens a short management connection and returns at minimum- ``{"slot_exists": bool, "publication_exists": bool, "lag_bytes": int | None}``.- Engines may add extra fields. Raises on connection failure — the caller- surfaces that as a 400 / unreachable state.+ Opens a short management connection and returns the engine's fields of the CDC status+ payload: its stored stream identifiers, whether each engine-side resource exists, and the+ lag. Raises on connection failure — the caller surfaces that as a 400 / unreachable state.+ """+ ...++ def capture_resources_present(self, status: Mapping[str, Any]) -> bool:+ """Whether a ``get_status`` result shows every engine-side resource capture reads from.++ False is the evidence that lets Repair CDC run, and that stops Resume CDC from unpausing a+ source that would only fail again."""...@@ -208,6 +218,12 @@ def cdc_supported_source_types() -> list[ExternalDataSourceType]:return list(_cdc_adapters().keys())+def cdc_engine(source_type: ExternalDataSourceType) -> CDCEngine | None:+ """The engine name of the source type's CDC adapter, or None when it has no change stream."""+ adapter = _cdc_adapters().get(source_type)+ return adapter.engine if adapter is not None else None++def source_type_supports_cdc(source_type: ExternalDataSourceType | str | None) -> bool:"""Whether the given source type (enum or raw string) supports CDC."""if source_type is None:diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.pyindex c3f4d6066815..064147c14311 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py@@ -216,7 +216,7 @@ def _require_broken_evidence(returnlive_status = adapter.get_status(source)- if live_status.get("slot_exists") is False or live_status.get("publication_exists") is False:+ if not adapter.capture_resources_present(live_status):returnraise CDCRepairError(diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.pyindex a982304c25ee..a14183d969c3 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py@@ -18,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformaTransaction,)from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig, CDCStreamReader, ChangeEvent+from products.warehouse_sources.backend.types import CDCEngineif TYPE_CHECKING:from contextlib import AbstractContextManager@@ -129,6 +130,9 @@ class FakeCDCStreamReader:class FakeCDCAdapter:+ # The catalog never lists the fake engine, so any member satisfies the interface.+ engine = CDCEngine.POSTGRES+def __init__(self, db: SourceDatabase) -> None:self._db = db@@ -200,6 +204,9 @@ class FakeCDCAdapter:def get_status(self, source: ExternalDataSource) -> dict[str, Any]:raise NotImplementedError(_NOT_EXERCISED)+ def capture_resources_present(self, status: Mapping[str, Any]) -> bool:+ raise NotImplementedError(_NOT_EXERCISED)+def add_table(self, source: ExternalDataSource, schema: str, table: str) -> None:raise NotImplementedError(_NOT_EXERCISED)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.pyindex b9392927d553..71d8503ef2eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py@@ -3,7 +3,7 @@from __future__ import annotationsimport logging-from collections.abc import Iterator+from collections.abc import Iterator, Mappingfrom contextlib import contextmanagerfrom typing import TYPE_CHECKING, Any, Literal@@ -41,6 +41,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.p_retry_on_connection_dropped,source_requires_ssl,)+from products.warehouse_sources.backend.types import CDCEngineif TYPE_CHECKING:from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource@@ -109,6 +110,8 @@ def _split_qualified_table(qualified: str, default_schema: str) -> tuple[str, stclass PostgresCDCAdapter:+ engine = CDCEngine.POSTGRES+def parse_cdc_config(self, source: ExternalDataSource) -> PostgresCDCConfig:return PostgresCDCConfig.from_source(source)@@ -378,12 +381,17 @@ class PostgresCDCAdapter:else [])return {+ "slot_name": cdc_config.slot_name,+ "publication_name": cdc_config.publication_name,"slot_exists": slot_present,"publication_exists": pub_present,"lag_bytes": lag_bytes,"published_tables": published_tables,}+ def capture_resources_present(self, status: Mapping[str, Any]) -> bool:+ return status.get("slot_exists") is not False and status.get("publication_exists") is not False+def add_table(self, source: ExternalDataSource, schema: str, table: str) -> None:"""ALTER PUBLICATION ADD TABLE. No-op for self-managed / no publication. Raises on failure."""self._alter_publication_membership(source, schema, table, add=True)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.pyindex 36f66be8fc0d..664b1b6108b6 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py@@ -404,6 +404,8 @@ class TestGetStatus:source = _source(cdc_enabled=True, cdc_slot_name="slot", cdc_publication_name="pub")status = PostgresCDCAdapter().get_status(source)assert status == {+ "slot_name": "slot",+ "publication_name": "pub","slot_exists": True,"publication_exists": True,"lag_bytes": 42,diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.pyindex b5f7181fa9fc..5d90672ee4a6 100644--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py@@ -12820,7 +12820,13 @@ class TestCDCStatus(APIBaseTest):)@patch("products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.get_status",- return_value={"slot_exists": True, "publication_exists": True, "lag_bytes": 2048},+ return_value={+ "slot_name": "posthog_slot",+ "publication_name": "posthog_pub",+ "slot_exists": True,+ "publication_exists": True,+ "lag_bytes": 2048,+ },)def test_returns_live_status_when_enabled(self, mock_get_status, _mock_paused) -> None:source = _make_postgres_source(self.team.pk, self.user, cdc_enabled=True)diff --git a/products/warehouse_sources/backend/tests/api/test_public_source_configs.py b/products/warehouse_sources/backend/tests/api/test_public_source_configs.pyindex 33ffbf06737f..a96f03116486 100644--- a/products/warehouse_sources/backend/tests/api/test_public_source_configs.py+++ b/products/warehouse_sources/backend/tests/api/test_public_source_configs.py@@ -91,6 +91,17 @@ class TestPublicSourceConfigs(APIBaseTest):response = self.client.get("/api/public_source_configs/")assert response.json()["Postgres"]["tables"] == []+ def test_cdc_engine_is_set_for_every_source_with_change_capture(self):+ response = self.client.get("/api/public_source_configs/")++ engines = {source_type: config["cdcEngine"] for source_type, config in response.json().items()}++ assert {source_type: engine for source_type, engine in engines.items() if engine is not None} == {+ "Postgres": "postgres",+ "Supabase": "postgres",+ "Neon": "postgres",+ }+def test_every_config_exposes_version_fields(self):"""The external version-update automation consumes these exact field names."""response = self.client.get("/api/public_source_configs/")diff --git a/products/warehouse_sources/backend/types.py b/products/warehouse_sources/backend/types.pyindex 265daebfec32..2c0f4e0e3da4 100644--- a/products/warehouse_sources/backend/types.py+++ b/products/warehouse_sources/backend/types.py@@ -3,6 +3,7 @@from products.warehouse_sources.backend.facade.types import (DIRECT_ENGINE_BY_SOURCE_TYPE as DIRECT_ENGINE_BY_SOURCE_TYPE,+ CDCEngine as CDCEngine,DataWarehouseManagedViewSetKind as DataWarehouseManagedViewSetKind,DataWarehouseTableCreatedVia as DataWarehouseTableCreatedVia,DataWarehouseTableFormat as DataWarehouseTableFormat,diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.tsindex 32ee6646e4cd..7a0d245ef220 100644--- a/products/warehouse_sources/frontend/generated/api.schemas.ts+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts@@ -14627,6 +14627,15 @@ export interface SourceVersionDeprecationApi {sunsetAt?: string | null}+/**+ * Change data capture engine of a source type, which picks its CDC setup form and status panel.+ */+export type CDCEngineApi = (typeof CDCEngineApi)[keyof typeof CDCEngineApi]++export const CDCEngineApi = {+ Postgres: 'postgres',+} as const+export interface SourceDocumentedTableApi {name: stringlabel: string@@ -14694,6 +14703,8 @@ export interface SourceConfigResponseApi {/** Vendor API docs or changelog URL, or null when the vendor publishes none. */apiDocsUrl?: string | nulldeprecatedVersions: SourceVersionDeprecationApi[]+ /** Change data capture engine, which picks the CDC setup form and status panel. Null when the source has no CDC. */+ cdcEngine?: CDCEngineApi | null/** Credential-free documented table catalog, empty for SQL and file sources with user-defined schemas. The public endpoint sets it; the wizard omits it to keep its payload small. */tables?: SourceDocumentedTableApi[] | null}diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.tsindex 600db7cea164..c67fe649a152 100644--- a/services/mcp/src/api/generated.ts+++ b/services/mcp/src/api/generated.ts@@ -20804,6 +20804,16 @@ export namespace Schemas {Other: 'other',} as const;+ /**+ * Change data capture engine of a source type, which picks its CDC setup form and status panel.+ */+ export type CDCEngine = typeof CDCEngine[keyof typeof CDCEngine];+++ export const CDCEngine = {+ Postgres: 'postgres',+ } as const;+export interface CICardSummary {/** Count of open pull requests. */open_prs: number;@@ -112079,6 +112089,8 @@ export namespace Schemas {/** Vendor API docs or changelog URL, or null when the vendor publishes none. */apiDocsUrl?: string | null;deprecatedVersions: SourceVersionDeprecation[];+ /** Change data capture engine, which picks the CDC setup form and status panel. Null when the source has no CDC. */+ cdcEngine?: CDCEngine | null;/** Credential-free documented table catalog, empty for SQL and file sources with user-defined schemas. The public endpoint sets it; the wizard omits it to keep its payload small. */tables?: SourceDocumentedTable[] | null;}