Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-warehouse): drive CDC setup and status gating from the adapter (#123574)

Daniel RC3f2b0a907112
diff --git a/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsx b/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsx
index 34e7048e7302..20a63917e46a 100644
--- a/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsx
+++ b/products/data_warehouse/frontend/scenes/SourceScene/tabs/CDCSection.tsx
@@ -32,7 +32,6 @@ import {
externalDataSourcesResumeCdcCreate,
} from 'products/warehouse_sources/frontend/generated/api'
-import { CDC_SOURCE_TYPES } from '../../../shared/cdc'
import { sourceSettingsLogic } from './sourceSettingsLogic'
type ManagementMode = 'posthog' | 'self_managed'
@@ -151,8 +150,9 @@ function confirmThen(opts: {
export function CDCSection({ source }: { source: ExternalDataSource }): JSX.Element | null {
const { featureFlags } = useValues(featureFlagLogic)
+ const { sourceFieldConfig } = useValues(sourceSettingsLogic)
- if (!CDC_SOURCE_TYPES.includes(source.source_type)) {
+ if (!sourceFieldConfig?.cdcEngine) {
return null
}
if (source.access_method !== 'warehouse') {
diff --git a/products/data_warehouse/frontend/shared/cdc.ts b/products/data_warehouse/frontend/shared/cdc.ts
deleted file mode 100644
index c136596c003e..000000000000
--- a/products/data_warehouse/frontend/shared/cdc.ts
+++ /dev/null
@@ -1,4 +0,0 @@
-// Source types that support change data capture (logical replication). Supabase and Neon
-// are Postgres on the wire and reuse the same CDC path. Keep in sync with the backend
-// adapter registry in products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py.
-export const CDC_SOURCE_TYPES: string[] = ['Postgres', 'Supabase', 'Neon']
diff --git a/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsx b/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsx
index 5ee692408d97..eb2cbabe3b4d 100644
--- a/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsx
+++ b/products/data_warehouse/frontend/shared/components/forms/SourceForm.tsx
@@ -35,7 +35,6 @@ import {
type SourceWizardLogicProps,
sourceWizardLogic,
} from '../../../scenes/NewSourceScene/sourceWizardLogic'
-import { CDC_SOURCE_TYPES } from '../../cdc'
import { CredentialAccountSelector } from './CredentialAccountSelector'
import { isCustomSourceAiBuilderEnabled } from './customSourceManifest'
import { CustomSourceManifestBuilder } from './CustomSourceManifestBuilder'
@@ -1030,7 +1029,7 @@ export function SourceFormComponent({
</Group>
{!isUpdateMode &&
showCdcConfig &&
- CDC_SOURCE_TYPES.includes(sourceConfig.name) &&
+ !!sourceConfig.cdcEngine &&
featureFlags[FEATURE_FLAGS.DWH_POSTGRES_CDC] &&
selectedAccessMethod === 'warehouse' && <CDCConfigSection sourceName={sourceConfig.name} />}
{showPrefix && !isDirectQuerySource && !customAiIntroActive && (
diff --git a/products/warehouse_sources/backend/facade/source_config.py b/products/warehouse_sources/backend/facade/source_config.py
index 70d4825418b2..a14dc182be6a 100644
--- a/products/warehouse_sources/backend/facade/source_config.py
+++ b/products/warehouse_sources/backend/facade/source_config.py
@@ -14,7 +14,7 @@ from typing import Annotated, Any, Literal, Union
from pydantic import BaseModel, ConfigDict, Field, RootModel
from pydantic.json_schema import SkipJsonSchema, WithJsonSchema
-from products.warehouse_sources.backend.facade.types import ExternalDataSourceType
+from products.warehouse_sources.backend.facade.types import CDCEngine, ExternalDataSourceType
__all__ = [
"DataWarehouseSourceCategory",
@@ -442,6 +442,13 @@ class SourceConfigResponse(SourceConfig):
description="Vendor API docs or changelog URL, or null when the vendor publishes none.",
)
deprecatedVersions: list[SourceVersionDeprecation]
+ cdcEngine: CDCEngine | None = Field(
+ default=None,
+ description=(
+ "Change data capture engine, which picks the CDC setup form and status panel. Null when"
+ " the source has no CDC."
+ ),
+ )
tables: list[SourceDocumentedTable] | None = Field(
default=None,
description=(
diff --git a/products/warehouse_sources/backend/facade/source_management.py b/products/warehouse_sources/backend/facade/source_management.py
index d4d5457ff0a3..3e36a91f6b1d 100644
--- a/products/warehouse_sources/backend/facade/source_management.py
+++ b/products/warehouse_sources/backend/facade/source_management.py
@@ -19,6 +19,7 @@ _LAZY = {
"fetch_docs_text": "sources.custom.ai_builder",
"CDCSourceAdapter": "cdc.adapters",
"add_table_failure_message": "cdc.adapters",
+ "cdc_engine": "cdc.adapters",
"get_cdc_adapter": "cdc.adapters",
"source_type_supports_cdc": "cdc.adapters",
"CDCRepairError": "cdc.repair",
diff --git a/products/warehouse_sources/backend/facade/types.py b/products/warehouse_sources/backend/facade/types.py
index 2faa7d7fcf26..03e4c7be30d8 100644
--- a/products/warehouse_sources/backend/facade/types.py
+++ b/products/warehouse_sources/backend/facade/types.py
@@ -1508,6 +1508,12 @@ DIRECT_ENGINE_BY_SOURCE_TYPE: dict[str, str] = {
}
+class CDCEngine(StrEnum):
+ """Change data capture engine of a source type, which picks its CDC setup form and status panel."""
+
+ POSTGRES = "postgres"
+
+
class ManagedWarehouseSQLMode(StrEnum):
EXTERNAL = "external"
BUILT_IN = "built_in"
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
index 3b11bf4c09ea..677dc8d9d7d5 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
@@ -720,7 +720,7 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
data={"message": f"Could not connect to source to resume CDC: {e}"},
)
- if live_status.get("slot_exists") is False or live_status.get("publication_exists") is False:
+ if not adapter.capture_resources_present(live_status):
return Response(
status=status.HTTP_400_BAD_REQUEST,
data={"message": "The replication slot or publication is missing. Use Repair CDC to recreate it."},
@@ -860,8 +860,6 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
{
"enabled": True,
"management_mode": cdc_config.management_mode,
- "slot_name": cdc_config.slot_name,
- "publication_name": cdc_config.publication_name,
"lag_warning_threshold_mb": cdc_config.lag_warning_threshold_mb,
"lag_critical_threshold_mb": cdc_config.lag_critical_threshold_mb,
"schedule_paused": schedule_paused,
diff --git a/products/warehouse_sources/backend/presentation/views/public_source_configs.py b/products/warehouse_sources/backend/presentation/views/public_source_configs.py
index d93683a9b277..b3805042aa2b 100644
--- a/products/warehouse_sources/backend/presentation/views/public_source_configs.py
+++ b/products/warehouse_sources/backend/presentation/views/public_source_configs.py
@@ -8,7 +8,7 @@ from rest_framework.request import Request
from rest_framework.response import Response
from products.warehouse_sources.backend.facade.source_config import SourceConfigMapResponse
-from products.warehouse_sources.backend.facade.source_management import SourceRegistry
+from products.warehouse_sources.backend.facade.source_management import SourceRegistry, cdc_engine
logger = structlog.get_logger(__name__)
tracer = trace.get_tracer(__name__)
@@ -42,6 +42,7 @@ def build_source_configs(*, include_tables: bool = True) -> dict[str, dict]:
config["versions"] = list(source.supported_versions)
config["defaultVersion"] = source.default_version
config["apiDocsUrl"] = source.api_docs_url
+ config["cdcEngine"] = cdc_engine(source_type)
config["deprecatedVersions"] = [
{"version": d.version, "sunsetAt": d.sunset_at.isoformat() if d.sunset_at else None}
for d in source.deprecated_versions
@@ -78,7 +79,8 @@ class PublicSourceConfigViewSet(viewsets.ViewSet):
"documented table catalog; empty for SQL/file sources with user-defined schemas), and "
"vendor API version metadata: `versions` (supported version labels), `defaultVersion`, "
"`apiDocsUrl` (vendor API docs/changelog URL or null), and `deprecatedVersions` "
- "(array of `{version, sunsetAt}` with `sunsetAt` an ISO date or null)."
+ "(array of `{version, sunsetAt}` with `sunsetAt` an ISO date or null), and `cdcEngine` "
+ "(the change data capture engine, or null when the source has no CDC)."
),
)
def list(self, request: Request) -> Response:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py
index 0e7a895cc24d..272e00d5e3e9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py
@@ -14,7 +14,7 @@ register it in ``_cdc_adapters`` below.
from __future__ import annotations
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar
@@ -22,7 +22,7 @@ from posthog.exceptions_capture import capture_exception
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapter
-from products.warehouse_sources.backend.types import ExternalDataSourceType
+from products.warehouse_sources.backend.types import CDCEngine, ExternalDataSourceType
if TYPE_CHECKING:
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
@@ -40,6 +40,9 @@ class CDCSourceAdapter(Protocol[CDCConfigT_co]):
is typed precisely without forcing callers to do ``isinstance`` checks.
"""
+ # The catalog serves it as `cdcEngine`. Engines that are the same database on the wire share one.
+ engine: CDCEngine
+
def create_reader(self, source: ExternalDataSource) -> CDCStreamReader: ...
def position_to_seq(self, position_serialized: str) -> int:
@@ -142,10 +145,17 @@ class CDCSourceAdapter(Protocol[CDCConfigT_co]):
def get_status(self, source: ExternalDataSource) -> dict[str, Any]:
"""Live engine-side CDC health for the source, read from the source DB.
- Opens a short management connection and returns at minimum
- ``{"slot_exists": bool, "publication_exists": bool, "lag_bytes": int | None}``.
- Engines may add extra fields. Raises on connection failure — the caller
- surfaces that as a 400 / unreachable state.
+ Opens a short management connection and returns the engine's fields of the CDC status
+ payload: its stored stream identifiers, whether each engine-side resource exists, and the
+ lag. Raises on connection failure — the caller surfaces that as a 400 / unreachable state.
+ """
+ ...
+
+ def capture_resources_present(self, status: Mapping[str, Any]) -> bool:
+ """Whether a ``get_status`` result shows every engine-side resource capture reads from.
+
+ False is the evidence that lets Repair CDC run, and that stops Resume CDC from unpausing a
+ source that would only fail again.
"""
...
@@ -208,6 +218,12 @@ def cdc_supported_source_types() -> list[ExternalDataSourceType]:
return list(_cdc_adapters().keys())
+def cdc_engine(source_type: ExternalDataSourceType) -> CDCEngine | None:
+ """The engine name of the source type's CDC adapter, or None when it has no change stream."""
+ adapter = _cdc_adapters().get(source_type)
+ return adapter.engine if adapter is not None else None
+
+
def source_type_supports_cdc(source_type: ExternalDataSourceType | str | None) -> bool:
"""Whether the given source type (enum or raw string) supports CDC."""
if source_type is None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py
index c3f4d6066815..064147c14311 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py
@@ -216,7 +216,7 @@ def _require_broken_evidence(
return
live_status = adapter.get_status(source)
- if live_status.get("slot_exists") is False or live_status.get("publication_exists") is False:
+ if not adapter.capture_resources_present(live_status):
return
raise CDCRepairError(
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
index a982304c25ee..a14183d969c3 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
@@ -18,6 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conforma
Transaction,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig, CDCStreamReader, ChangeEvent
+from products.warehouse_sources.backend.types import CDCEngine
if TYPE_CHECKING:
from contextlib import AbstractContextManager
@@ -129,6 +130,9 @@ class FakeCDCStreamReader:
class FakeCDCAdapter:
+ # The catalog never lists the fake engine, so any member satisfies the interface.
+ engine = CDCEngine.POSTGRES
+
def __init__(self, db: SourceDatabase) -> None:
self._db = db
@@ -200,6 +204,9 @@ class FakeCDCAdapter:
def get_status(self, source: ExternalDataSource) -> dict[str, Any]:
raise NotImplementedError(_NOT_EXERCISED)
+ def capture_resources_present(self, status: Mapping[str, Any]) -> bool:
+ raise NotImplementedError(_NOT_EXERCISED)
+
def add_table(self, source: ExternalDataSource, schema: str, table: str) -> None:
raise NotImplementedError(_NOT_EXERCISED)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
index b9392927d553..71d8503ef2eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
@@ -3,7 +3,7 @@
from __future__ import annotations
import logging
-from collections.abc import Iterator
+from collections.abc import Iterator, Mapping
from contextlib import contextmanager
from typing import TYPE_CHECKING, Any, Literal
@@ -41,6 +41,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.p
_retry_on_connection_dropped,
source_requires_ssl,
)
+from products.warehouse_sources.backend.types import CDCEngine
if TYPE_CHECKING:
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
@@ -109,6 +110,8 @@ def _split_qualified_table(qualified: str, default_schema: str) -> tuple[str, st
class PostgresCDCAdapter:
+ engine = CDCEngine.POSTGRES
+
def parse_cdc_config(self, source: ExternalDataSource) -> PostgresCDCConfig:
return PostgresCDCConfig.from_source(source)
@@ -378,12 +381,17 @@ class PostgresCDCAdapter:
else []
)
return {
+ "slot_name": cdc_config.slot_name,
+ "publication_name": cdc_config.publication_name,
"slot_exists": slot_present,
"publication_exists": pub_present,
"lag_bytes": lag_bytes,
"published_tables": published_tables,
}
+ def capture_resources_present(self, status: Mapping[str, Any]) -> bool:
+ return status.get("slot_exists") is not False and status.get("publication_exists") is not False
+
def add_table(self, source: ExternalDataSource, schema: str, table: str) -> None:
"""ALTER PUBLICATION ADD TABLE. No-op for self-managed / no publication. Raises on failure."""
self._alter_publication_membership(source, schema, table, add=True)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py
index 36f66be8fc0d..664b1b6108b6 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py
@@ -404,6 +404,8 @@ class TestGetStatus:
source = _source(cdc_enabled=True, cdc_slot_name="slot", cdc_publication_name="pub")
status = PostgresCDCAdapter().get_status(source)
assert status == {
+ "slot_name": "slot",
+ "publication_name": "pub",
"slot_exists": True,
"publication_exists": True,
"lag_bytes": 42,
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index b5f7181fa9fc..5d90672ee4a6 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -12820,7 +12820,13 @@ class TestCDCStatus(APIBaseTest):
)
@patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.get_status",
- return_value={"slot_exists": True, "publication_exists": True, "lag_bytes": 2048},
+ return_value={
+ "slot_name": "posthog_slot",
+ "publication_name": "posthog_pub",
+ "slot_exists": True,
+ "publication_exists": True,
+ "lag_bytes": 2048,
+ },
)
def test_returns_live_status_when_enabled(self, mock_get_status, _mock_paused) -> None:
source = _make_postgres_source(self.team.pk, self.user, cdc_enabled=True)
diff --git a/products/warehouse_sources/backend/tests/api/test_public_source_configs.py b/products/warehouse_sources/backend/tests/api/test_public_source_configs.py
index 33ffbf06737f..a96f03116486 100644
--- a/products/warehouse_sources/backend/tests/api/test_public_source_configs.py
+++ b/products/warehouse_sources/backend/tests/api/test_public_source_configs.py
@@ -91,6 +91,17 @@ class TestPublicSourceConfigs(APIBaseTest):
response = self.client.get("/api/public_source_configs/")
assert response.json()["Postgres"]["tables"] == []
+ def test_cdc_engine_is_set_for_every_source_with_change_capture(self):
+ response = self.client.get("/api/public_source_configs/")
+
+ engines = {source_type: config["cdcEngine"] for source_type, config in response.json().items()}
+
+ assert {source_type: engine for source_type, engine in engines.items() if engine is not None} == {
+ "Postgres": "postgres",
+ "Supabase": "postgres",
+ "Neon": "postgres",
+ }
+
def test_every_config_exposes_version_fields(self):
"""The external version-update automation consumes these exact field names."""
response = self.client.get("/api/public_source_configs/")
diff --git a/products/warehouse_sources/backend/types.py b/products/warehouse_sources/backend/types.py
index 265daebfec32..2c0f4e0e3da4 100644
--- a/products/warehouse_sources/backend/types.py
+++ b/products/warehouse_sources/backend/types.py
@@ -3,6 +3,7 @@
from products.warehouse_sources.backend.facade.types import (
DIRECT_ENGINE_BY_SOURCE_TYPE as DIRECT_ENGINE_BY_SOURCE_TYPE,
+ CDCEngine as CDCEngine,
DataWarehouseManagedViewSetKind as DataWarehouseManagedViewSetKind,
DataWarehouseTableCreatedVia as DataWarehouseTableCreatedVia,
DataWarehouseTableFormat as DataWarehouseTableFormat,
diff --git a/products/warehouse_sources/frontend/generated/api.schemas.ts b/products/warehouse_sources/frontend/generated/api.schemas.ts
index 32ee6646e4cd..7a0d245ef220 100644
--- a/products/warehouse_sources/frontend/generated/api.schemas.ts
+++ b/products/warehouse_sources/frontend/generated/api.schemas.ts
@@ -14627,6 +14627,15 @@ export interface SourceVersionDeprecationApi {
sunsetAt?: string | null
}
+/**
+ * Change data capture engine of a source type, which picks its CDC setup form and status panel.
+ */
+export type CDCEngineApi = (typeof CDCEngineApi)[keyof typeof CDCEngineApi]
+
+export const CDCEngineApi = {
+ Postgres: 'postgres',
+} as const
+
export interface SourceDocumentedTableApi {
name: string
label: string
@@ -14694,6 +14703,8 @@ export interface SourceConfigResponseApi {
/** Vendor API docs or changelog URL, or null when the vendor publishes none. */
apiDocsUrl?: string | null
deprecatedVersions: SourceVersionDeprecationApi[]
+ /** Change data capture engine, which picks the CDC setup form and status panel. Null when the source has no CDC. */
+ cdcEngine?: CDCEngineApi | null
/** Credential-free documented table catalog, empty for SQL and file sources with user-defined schemas. The public endpoint sets it; the wizard omits it to keep its payload small. */
tables?: SourceDocumentedTableApi[] | null
}
diff --git a/services/mcp/src/api/generated.ts b/services/mcp/src/api/generated.ts
index 600db7cea164..c67fe649a152 100644
--- a/services/mcp/src/api/generated.ts
+++ b/services/mcp/src/api/generated.ts
@@ -20804,6 +20804,16 @@ export namespace Schemas {
Other: 'other',
} as const;
+ /**
+ * Change data capture engine of a source type, which picks its CDC setup form and status panel.
+ */
+ export type CDCEngine = typeof CDCEngine[keyof typeof CDCEngine];
+
+
+ export const CDCEngine = {
+ Postgres: 'postgres',
+ } as const;
+
export interface CICardSummary {
/** Count of open pull requests. */
open_prs: number;
@@ -112079,6 +112089,8 @@ export namespace Schemas {
/** Vendor API docs or changelog URL, or null when the vendor publishes none. */
apiDocsUrl?: string | null;
deprecatedVersions: SourceVersionDeprecation[];
+ /** Change data capture engine, which picks the CDC setup form and status panel. Null when the source has no CDC. */
+ cdcEngine?: CDCEngine | null;
/** Credential-free documented table catalog, empty for SQL and file sources with user-defined schemas. The public endpoint sets it; the wizard omits it to keep its payload small. */
tables?: SourceDocumentedTable[] | null;
}