Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-warehouse): give the CDC adapter interface stream-neutral names (#123575)

Daniel RCcb78fc13b4b2
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_schema.py b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
index 9685c0632e95..2a3a82d170f4 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_schema.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_schema.py
@@ -1966,8 +1966,7 @@ class ExternalDataSchemaSerializer(UserAccessControlSerializerMixin, serializers
changes made in the meantime would never reach it.
"""
adapter = get_cdc_adapter(source)
- cdc_config = adapter.parse_cdc_config(source)
- if cdc_config.management_mode != "posthog" or not cdc_config.publication_name:
+ if not adapter.manages_capture_set(source):
return None
_, db_schema, source_table_name = get_postgres_source_location(
diff --git a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
index 677dc8d9d7d5..d78078f82ddb 100644
--- a/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
+++ b/products/warehouse_sources/backend/presentation/views/external_data_source/change_data_capture.py
@@ -226,16 +226,13 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
)
tables = request.data.get("tables") or []
- slot_name = request.data.get("cdc_slot_name") or None
- publication_name = request.data.get("cdc_publication_name") or None
try:
prereq_errors = source_impl.check_cdc_prerequisites(
config,
management_mode=management_mode,
tables=tables,
- slot_name=slot_name,
- publication_name=publication_name,
+ setup=request.data,
team_id=self.team_id,
)
except base._EXPECTED_CONNECTION_ERRORS as e:
@@ -304,8 +301,7 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
management_mode=management_mode,
tables=[],
schema=schema_hint,
- slot_name=request.data.get("cdc_slot_name") or None,
- publication_name=request.data.get("cdc_publication_name") or None,
+ setup=request.data,
)
except base._EXPECTED_CONNECTION_ERRORS as e:
# Probing the source's database to validate it is expected to fail when the host,
@@ -380,8 +376,7 @@ class ExternalDataSourceCDCMixin(base.ExternalDataSourceViewSetBase):
management_mode=management_mode,
tables=[],
schema=schema_hint,
- slot_name=request.data.get("cdc_slot_name") or None,
- publication_name=request.data.get("cdc_publication_name") or None,
+ setup=request.data,
)
except base._EXPECTED_CONNECTION_ERRORS as e:
# Expected user/upstream connection failure (bad host/credentials/SSH tunnel, server
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
index d41a78b110a5..7badfcdd00ee 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/activities.py
@@ -208,7 +208,7 @@ class CDCExtractActivity:
# Transaction-boundary tracking for safe mid-run slot advances. All events of
# one transaction share the same commit end LSN (position_serialized), so an LSN
# change marks a fully-yielded transaction. We only ever micro-advance the slot
- # past transactions that are completely yielded — see _read_wal_loop.
+ # past transactions that are completely yielded — see _read_changes_loop.
self.current_txn_lsn: str | None = None
self.last_complete_txn_end_lsn: str | None = None
# End of WAL before this run peeked, and whether the peek reached the end of the backlog.
@@ -426,7 +426,7 @@ class CDCExtractActivity:
self._pre_read_position = self.reader.current_position()
self._load_pk_columns()
- self._read_wal_loop()
+ self._read_changes_loop()
# A read that got this far proves the slot is back, so a reset held for it can finish on
# the next run even if recovery never runs again.
for schema in self.cdc_schemas:
@@ -744,7 +744,7 @@ class CDCExtractActivity:
return heartbeat
- def _read_wal_loop(self) -> None:
+ def _read_changes_loop(self) -> None:
"""Read WAL events with periodic micro-batch flushes, bounded per peek.
Each pass peeks at most ``CDC_MAX_CHANGES_PER_READ`` changes so a large backlog can't
@@ -874,7 +874,7 @@ class CDCExtractActivity:
self._process_flush(tables)
metrics.get_micro_batches_flushed_metric(self.inputs.team_id, str(self.inputs.source_id)).add(1)
- commit_lsn = self.reader.last_commit_end_lsn
+ commit_lsn = self.reader.last_commit_position
if commit_lsn is not None and commit_lsn != self.last_confirmed_lsn:
self._handle_truncates()
self._confirm_position(commit_lsn)
@@ -1121,7 +1121,7 @@ class CDCExtractActivity:
# A mid-run page advance already released the WAL it committed, so the slot moved this run.
advanced = self.last_confirmed_lsn is not None
if truncated_tables:
- truncate_end_lsn = self.reader.last_commit_end_lsn
+ truncate_end_lsn = self.reader.last_commit_position
if truncate_end_lsn is not None:
self._confirm_position(truncate_end_lsn)
self.log.info("slot_advanced_past_truncate", position=truncate_end_lsn)
@@ -1168,7 +1168,7 @@ class CDCExtractActivity:
trailing transactions with no row events. A TRUNCATE on its own is one: this run already
handled it, and reading it again would reset the table a second time.
"""
- target = self.reader.last_commit_end_lsn or self.last_end_lsn
+ target = self.reader.last_commit_position or self.last_end_lsn
if target is not None and target != self.last_confirmed_lsn:
self._confirm_position(target)
self.last_end_lsn = target
@@ -1193,7 +1193,7 @@ class CDCExtractActivity:
"""
assert self.adapter is not None
assert self.source is not None
- if not self.adapter.parse_cdc_config(self.source).slot_name:
+ if not self.adapter.parse_cdc_config(self.source).stream_name:
raise CDCSlotNotConfiguredError()
# ------------------------------------------------------------------
@@ -1226,7 +1226,7 @@ class CDCExtractActivity:
schema.save(update_fields=["status", "latest_error", "updated_at"])
self._schema_log(schema).warning("cdc_schema_reset_for_slot_recovery", schema_id=str(schema.id))
- resource_fields = self.adapter.recreate_slot(
+ resource_fields = self.adapter.recreate_stream(
self.source, tables=[self._qualified_table_name(s) for s in self.cdc_schemas]
)
@@ -1511,8 +1511,7 @@ def validate_cdc_prerequisites_activity(inputs: ValidateCDCPrerequisitesInput) -
management_mode=inputs.management_mode, # type: ignore[arg-type]
tables=inputs.tables,
schema=inputs.schema,
- slot_name=inputs.slot_name,
- publication_name=inputs.publication_name,
+ setup={"cdc_slot_name": inputs.slot_name, "cdc_publication_name": inputs.publication_name},
)
@@ -1568,7 +1567,7 @@ def cleanup_orphan_slots_activity() -> None:
# Restore the original filter semantics on decrypted values: skip sources that
# don't have CDC enabled with both a slot and publication name to clean up.
- if not (cdc_config.enabled and cdc_config.slot_name and cdc_config.publication_name):
+ if not (cdc_config.enabled and cdc_config.stream_resources_configured):
continue
sources_checked += 1
@@ -1576,7 +1575,7 @@ def cleanup_orphan_slots_activity() -> None:
source_log = log.bind(
source_id=str(source.id),
team_id=source.team_id,
- slot_name=cdc_config.slot_name,
+ slot_name=cdc_config.stream_name,
management_mode=cdc_config.management_mode,
)
@@ -1595,7 +1594,7 @@ def cleanup_orphan_slots_activity() -> None:
source_log.info("cleaning_up_deleted_source_slot")
try:
with adapter.management_connection(source, connect_timeout=10) as conn:
- adapter.drop_resources(conn, cdc_config.slot_name, cdc_config.publication_name)
+ adapter.drop_stream(conn, source)
slots_dropped += 1
except Exception:
source_log.exception("failed_to_cleanup_deleted_source_slot")
@@ -1639,20 +1638,20 @@ def cleanup_orphan_slots_activity() -> None:
source_started = dt.datetime.now(tz=dt.UTC)
try:
with adapter.management_connection(source, connect_timeout=10) as conn:
- lag_bytes = adapter.get_lag_bytes(conn, cdc_config.slot_name)
- retention_cap_mb = adapter.get_retention_cap_mb(conn)
+ lag = adapter.get_lag(conn, source)
except Exception:
source_log.exception("failed_to_check_slot_lag")
metrics.get_sweeper_source_errors_metric().add(1)
sources_errored += 1
continue
- if lag_bytes is None:
+ if lag is None:
source_log.warning("slot_not_found_or_no_flush_lsn")
continue
- metrics.get_wal_lag_metric(source.team_id, str(source.id)).set(lag_bytes)
- lag_mb = lag_bytes / (1024 * 1024)
+ metrics.get_wal_lag_metric(source.team_id, str(source.id)).set(lag.behind_bytes)
+ lag_mb = lag.behind_bytes / (1024 * 1024)
+ retention_cap_mb = lag.retention_cap_mb
critical_threshold_mb = cdc_config.lag_critical_threshold_mb
if retention_cap_mb is not None:
@@ -1670,13 +1669,13 @@ def cleanup_orphan_slots_activity() -> None:
source_log.warning("auto_dropping_slot_critical_lag")
try:
with adapter.management_connection(source, connect_timeout=10) as conn:
- adapter.drop_resources(conn, cdc_config.slot_name, cdc_config.publication_name)
- # drop_resources is best-effort, so it logs a refused drop (an active slot, a missing
+ adapter.drop_stream(conn, source)
+ # drop_stream is best-effort, so it logs a refused drop (an active slot, a missing
# grant) instead of raising. Pausing capture then would leave nothing to advance the
# slot, so the source keeps running and the next sweep retries the drop.
- if adapter.slot_exists(conn, cdc_config.slot_name):
+ if adapter.stream_exists(conn, source):
raise RuntimeError(
- f"Replication slot {cdc_config.slot_name} still exists after the drop"
+ f"Replication slot {cdc_config.stream_name} still exists after the drop"
)
slots_dropped += 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py
index 272e00d5e3e9..295b07fada7b 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/adapters.py
@@ -20,7 +20,7 @@ from typing import TYPE_CHECKING, Any, Literal, Protocol, TypeVar
from posthog.exceptions_capture import capture_exception
-from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig, CDCLag
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapter
from products.warehouse_sources.backend.types import CDCEngine, ExternalDataSourceType
@@ -60,24 +60,25 @@ class CDCSourceAdapter(Protocol[CDCConfigT_co]):
management_mode: Literal["posthog", "self_managed"],
tables: list[str],
schema: str,
- slot_name: str | None,
- publication_name: str | None,
- ) -> list[str]: ...
+ setup: Mapping[str, Any],
+ ) -> list[str]:
+ """Check the source database against the engine's CDC prerequisites. ``setup`` holds the
+ engine's own ``cdc_*`` setup fields from the request (PG: the slot and publication names)."""
+ ...
- def drop_resources(self, conn: Any, slot_name: str, pub_name: str) -> None: ...
+ def drop_stream(self, conn: Any, source: ExternalDataSource) -> None:
+ """Best-effort drop of the source's engine-side change stream and the resources PostHog owns
+ with it (PG: the slot and the publication). Logs a refused drop instead of raising."""
+ ...
- def slot_exists(self, conn: Any, slot_name: str) -> bool:
+ def stream_exists(self, conn: Any, source: ExternalDataSource) -> bool:
"""Whether the change-stream resource still exists on the source database. Callers use it to
confirm a best-effort drop actually removed it before acting as if it were gone."""
...
- def get_lag_bytes(self, conn: Any, slot_name: str) -> int | None: ...
-
- def get_retention_cap_mb(self, conn: Any) -> int | None:
- """Engine-enforced cap on retained change-stream backlog in MB (PG:
- max_slot_wal_keep_size). None when unlimited or unknown. Once the backlog
- crosses this cap the engine invalidates the slot itself, so safety nets
- must act below it."""
+ def get_lag(self, conn: Any, source: ExternalDataSource) -> CDCLag | None:
+ """How far capture is behind the change stream, or None when the stream is missing or reports
+ no position."""
...
def is_slot_invalidation_error(self, exc: BaseException) -> bool:
@@ -106,7 +107,7 @@ class CDCSourceAdapter(Protocol[CDCConfigT_co]):
``is_slot_invalidation_error``: keeps psycopg/engine specifics out of the shared layer."""
...
- def recreate_slot(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:
+ def recreate_stream(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:
"""Drop and recreate the change-stream resource after invalidation, against the
existing capture definition (recreating it when PostHog owns it). ``tables`` is
the schema-qualified (``schema.table``) capture set used if the definition must be
@@ -171,6 +172,11 @@ class CDCSourceAdapter(Protocol[CDCConfigT_co]):
"""Best-effort exclude a table from the change-capture set. Inverse of ``add_table``."""
...
+ def manages_capture_set(self, source: ExternalDataSource) -> bool:
+ """Whether PostHog owns the source's change-capture set (PG: a PostHog-managed publication),
+ so ``add_table`` and ``remove_table`` change it."""
+ ...
+
def _cdc_adapters() -> dict[ExternalDataSourceType, CDCSourceAdapter[CDCConfig]]:
"""Registry of CDC adapters keyed by source type. Adding a new CDC-capable source
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/billing_expiry.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/billing_expiry.py
index e38223759e1f..a62fe9314dbf 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/billing_expiry.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/billing_expiry.py
@@ -129,11 +129,11 @@ def stop_cdc_past_billing_retention(
"""
if cdc_config.management_mode == "posthog" and cdc_config.auto_drop_slot:
with adapter.management_connection(source, connect_timeout=10) as conn:
- adapter.drop_resources(conn, cdc_config.slot_name, cdc_config.publication_name)
- # drop_resources is best-effort: it logs a refused drop (an active slot, a missing grant)
+ adapter.drop_stream(conn, source)
+ # drop_stream is best-effort: it logs a refused drop (an active slot, a missing grant)
# instead of raising.
- if adapter.slot_exists(conn, cdc_config.slot_name):
- raise RuntimeError(f"Replication slot {cdc_config.slot_name} still exists after the drop")
+ if adapter.stream_exists(conn, source):
+ raise RuntimeError(f"Replication slot {cdc_config.stream_name} still exists after the drop")
mark_cdc_broken(source, BILLING_LIMIT_EXPIRED_REASON, _SLOT_DROPPED_MESSAGE)
return True
mark_cdc_broken(source, BILLING_LIMIT_EXPIRED_REASON, _SLOT_KEPT_MESSAGE, pause=False, slot_kept=True)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py
index 064147c14311..73a3a72acfeb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/repair.py
@@ -125,7 +125,7 @@ def _repair_locked(source: ExternalDataSource) -> int:
# A sync that hands over after its table's reset leaves the reset pending on a streaming table,
# whose next run wipes it. Capture finishes those resets once the syncs stop, after this repair.
# They wait for the slot as well, because the one they would snapshot against is gone until
- # `recreate_slot` below, and a capture run firing meanwhile must not start that snapshot.
+ # `recreate_stream` below, and a capture run firing meanwhile must not start that snapshot.
handed_over = {
schema.id
for schema in all_cdc_schemas
@@ -155,7 +155,7 @@ def _repair_locked(source: ExternalDataSource) -> int:
default_schema = SourceInputsStore.load_settings(source).get("schema")
try:
- resource_fields = adapter.recreate_slot(
+ resource_fields = adapter.recreate_stream(
source, tables=[cdc_qualified_table_name(schema, default_schema) for schema in cdc_schemas]
)
except Exception as e:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py
index f6a0ece85011..cecc7f5737a1 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py
@@ -3,7 +3,8 @@
from __future__ import annotations
import logging
-from typing import TYPE_CHECKING
+from collections.abc import Mapping
+from typing import TYPE_CHECKING, Any
from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import (
get_cdc_adapter,
@@ -32,14 +33,14 @@ class CDCSourceMixin(_BaseSource[ConfigType]):
config: ConfigType,
management_mode: str,
tables: list[str],
- slot_name: str | None = None,
- publication_name: str | None = None,
+ setup: Mapping[str, Any],
require_ssl: bool = True,
team_id: int | None = None,
) -> list[str]:
"""Validate the CDC prerequisites against a live connection, before the source exists.
- Returns one message per unmet prerequisite.
+ ``setup`` holds the engine's own ``cdc_*`` setup fields from the request. Returns one
+ message per unmet prerequisite.
"""
raise NotImplementedError(f"{type(self).__name__} does not check CDC prerequisites")
@@ -68,16 +69,16 @@ class CDCSourceMixin(_BaseSource[ConfigType]):
if cdc_config.management_mode != "posthog":
return
- if not cdc_config.slot_name or not cdc_config.publication_name:
+ if not cdc_config.stream_resources_configured:
return
try:
with adapter.management_connection(source, connect_timeout=10) as conn:
- adapter.drop_resources(conn, cdc_config.slot_name, cdc_config.publication_name)
+ adapter.drop_stream(conn, source)
except Exception:
log.exception(
"Failed to drop CDC slot/publication on source DB (best-effort)",
- extra={"source_id": str(source.id), "slot_name": cdc_config.slot_name},
+ extra={"source_id": str(source.id), "slot_name": cdc_config.stream_name},
)
def _buffered_cdc_source(self, schema: ExternalDataSchema, inputs: SourceInputs) -> SourceResponse:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
index a14183d969c3..cf89f03f7eff 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
@@ -8,6 +8,7 @@ from __future__ import annotations
from collections.abc import Callable, Iterable, Iterator, Mapping
from contextlib import contextmanager, nullcontext
+from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Literal
from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import CDCSourceAdapter
@@ -17,7 +18,12 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conforma
SourceDatabase,
Transaction,
)
-from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig, CDCStreamReader, ChangeEvent
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import (
+ CDCConfig,
+ CDCLag,
+ CDCStreamReader,
+ ChangeEvent,
+)
from products.warehouse_sources.backend.types import CDCEngine
if TYPE_CHECKING:
@@ -26,7 +32,6 @@ if TYPE_CHECKING:
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
_STREAM_NAME = "fake_stream"
-_CAPTURE_SET_NAME = "fake_capture_set"
# Raised by the adapter methods that only setup, the sweeper and repair call, so a test that starts
# to reach one fails instead of trusting behavior nothing checks.
_NOT_EXERCISED = "The conformance suite does not exercise this adapter method"
@@ -36,6 +41,17 @@ class FakeStreamLostError(Exception):
pass
+@dataclass(frozen=True)
+class FakeCDCConfig(CDCConfig):
+ @property
+ def stream_name(self) -> str:
+ return _STREAM_NAME
+
+ @property
+ def stream_resources_configured(self) -> bool:
+ return True
+
+
def _position(transaction_number: int) -> str:
return f"{transaction_number:012d}"
@@ -122,7 +138,7 @@ class FakeCDCStreamReader:
self._truncated.clear()
@property
- def last_commit_end_lsn(self) -> str | None:
+ def last_commit_position(self) -> str | None:
return self._last_commit_position
def close(self) -> None:
@@ -152,21 +168,17 @@ class FakeCDCAdapter:
management_mode: Literal["posthog", "self_managed"],
tables: list[str],
schema: str,
- slot_name: str | None,
- publication_name: str | None,
+ setup: Mapping[str, Any],
) -> list[str]:
raise NotImplementedError(_NOT_EXERCISED)
- def drop_resources(self, conn: Any, slot_name: str, pub_name: str) -> None:
- raise NotImplementedError(_NOT_EXERCISED)
-
- def slot_exists(self, conn: Any, slot_name: str) -> bool:
+ def drop_stream(self, conn: Any, source: ExternalDataSource) -> None:
raise NotImplementedError(_NOT_EXERCISED)
- def get_lag_bytes(self, conn: Any, slot_name: str) -> int | None:
+ def stream_exists(self, conn: Any, source: ExternalDataSource) -> bool:
raise NotImplementedError(_NOT_EXERCISED)
- def get_retention_cap_mb(self, conn: Any) -> int | None:
+ def get_lag(self, conn: Any, source: ExternalDataSource) -> CDCLag | None:
raise NotImplementedError(_NOT_EXERCISED)
def is_slot_invalidation_error(self, exc: BaseException) -> bool:
@@ -181,14 +193,12 @@ class FakeCDCAdapter:
def classify_error(self, exc: BaseException) -> CDCErrorInfo | None:
return None
- def recreate_slot(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:
+ def recreate_stream(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:
return {"cdc_consistent_point": _position(self._db.recreate_stream())}
def parse_cdc_config(self, source: ExternalDataSource) -> CDCConfig:
- return CDCConfig(
+ return FakeCDCConfig(
enabled=True,
- slot_name=_STREAM_NAME,
- publication_name=_CAPTURE_SET_NAME,
management_mode="posthog",
lag_warning_threshold_mb=512,
lag_critical_threshold_mb=2048,
@@ -213,6 +223,9 @@ class FakeCDCAdapter:
def remove_table(self, source: ExternalDataSource, schema: str, table: str) -> None:
raise NotImplementedError(_NOT_EXERCISED)
+ def manages_capture_set(self, source: ExternalDataSource) -> bool:
+ raise NotImplementedError(_NOT_EXERCISED)
+
class FakeEngine:
source_type = "FakeEngine"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_cleanup_orphan_slots.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_cleanup_orphan_slots.py
index 69edd78573dc..95a1a7ac64b0 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_cleanup_orphan_slots.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_cleanup_orphan_slots.py
@@ -9,6 +9,7 @@ from products.warehouse_sources.backend.models.external_data_job import External
from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema
from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
from products.warehouse_sources.backend.temporal.data_imports.cdc.activities import cleanup_orphan_slots_activity
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCLag
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.config import PostgresCDCConfig
pytestmark = pytest.mark.django_db
@@ -61,9 +62,10 @@ def _mock_adapter(*, lag_bytes=0, retention_cap_mb=None, slot_survives_drop=Fals
yield MagicMock()
adapter.management_connection.side_effect = _conn
- adapter.get_lag_bytes.return_value = lag_bytes
- adapter.get_retention_cap_mb.return_value = retention_cap_mb
- adapter.slot_exists.return_value = slot_survives_drop
+ adapter.get_lag.return_value = (
+ None if lag_bytes is None else CDCLag(behind_bytes=lag_bytes, retention_cap_mb=retention_cap_mb)
+ )
+ adapter.stream_exists.return_value = slot_survives_drop
return adapter
@@ -95,7 +97,7 @@ def test_encrypted_cdc_source_is_selected(team):
_run(adapter)
- adapter.get_lag_bytes.assert_called_once()
+ adapter.get_lag.assert_called_once()
@pytest.mark.parametrize(
@@ -113,7 +115,7 @@ def test_disabled_or_incomplete_sources_skipped(team, job_inputs):
_run(adapter)
- adapter.get_lag_bytes.assert_not_called()
+ adapter.get_lag.assert_not_called()
def test_deleted_posthog_managed_drops_resources_and_schedule(team):
@@ -123,8 +125,8 @@ def test_deleted_posthog_managed_drops_resources_and_schedule(team):
_, mock_delete, _ = _run(adapter)
mock_delete.assert_called_once()
- adapter.drop_resources.assert_called_once()
- adapter.get_lag_bytes.assert_not_called()
+ adapter.drop_stream.assert_called_once()
+ adapter.get_lag.assert_not_called()
def test_deleted_self_managed_drops_schedule_but_not_slot(team):
@@ -134,7 +136,7 @@ def test_deleted_self_managed_drops_schedule_but_not_slot(team):
_, mock_delete, _ = _run(adapter)
mock_delete.assert_called_once()
- adapter.drop_resources.assert_not_called()
+ adapter.drop_stream.assert_not_called()
def test_deleted_source_purges_shadow_buffer_prefixes(team):
@@ -165,7 +167,7 @@ def test_critical_lag_posthog_auto_drop_stops_capture_once_the_slot_is_gone(team
_, _, mock_pause = _run(adapter)
- adapter.drop_resources.assert_called_once()
+ adapter.drop_stream.assert_called_once()
source.refresh_from_db()
assert (source.status == ExternalDataSource.Status.ERROR) is stopped
schema.refresh_from_db()
@@ -208,7 +210,7 @@ def test_critical_lag_auto_drop_disabled_does_not_drop(team):
_run(adapter)
- adapter.drop_resources.assert_not_called()
+ adapter.drop_stream.assert_not_called()
@pytest.mark.parametrize(
@@ -234,7 +236,7 @@ def test_critical_lag_self_managed_marks_broken_without_drop_or_pause(
_, _, mock_pause = _run(adapter)
# Customer owns the slot: surface the broken state but never drop or pause — it may recover.
- adapter.drop_resources.assert_not_called()
+ adapter.drop_stream.assert_not_called()
mock_pause.assert_not_called()
source.refresh_from_db()
assert source.status == expected_status
@@ -277,11 +279,11 @@ def test_a_source_blocked_by_billing_past_buffer_retention_is_marked_broken(team
with patch(f"{_BILLING_EXPIRY}.is_team_limited", return_value=True):
_, _, mock_pause = _run(adapter)
- assert adapter.drop_resources.called is dropped
+ assert adapter.drop_stream.called is dropped
assert mock_pause.called is paused
schema.refresh_from_db()
assert schema.sync_type_config["cdc_broken"]["reason"] == "billing_limit_expired"
- adapter.get_lag_bytes.assert_not_called()
+ adapter.get_lag.assert_not_called()
@pytest.mark.parametrize(
@@ -309,13 +311,13 @@ def test_a_source_is_left_running_unless_billing_blocks_it_past_buffer_retention
with patch(f"{_BILLING_EXPIRY}.is_team_limited", return_value=team_limited):
_run(adapter)
- adapter.drop_resources.assert_not_called()
+ adapter.drop_stream.assert_not_called()
schema.refresh_from_db()
assert (schema.sync_type_config.get("cdc_broken") or {}).get("reason") != "billing_limit_expired"
def test_a_slot_that_survives_the_drop_keeps_billing_capture_running(team):
- # drop_resources only logs a refused drop, so pausing capture on the strength of the call alone
+ # drop_stream only logs a refused drop, so pausing capture on the strength of the call alone
# would leave a live slot with nothing advancing it and the customer's WAL growing.
source = _create_source(team, job_inputs=_cdc_job_inputs())
schema = _billing_blocked_schema(team, source, blocked_for=dt.timedelta(days=15))
@@ -324,9 +326,9 @@ def test_a_slot_that_survives_the_drop_keeps_billing_capture_running(team):
with patch(f"{_BILLING_EXPIRY}.is_team_limited", return_value=True):
_, _, mock_pause = _run(adapter)
- adapter.drop_resources.assert_called()
+ adapter.drop_stream.assert_called()
mock_pause.assert_not_called()
- adapter.get_lag_bytes.assert_called_once()
+ adapter.get_lag.assert_called_once()
source.refresh_from_db()
assert source.status != ExternalDataSource.Status.ERROR
schema.refresh_from_db()
@@ -354,7 +356,7 @@ def test_a_job_that_is_not_the_tables_own_sync_does_not_defer_the_billing_stop(t
with patch(f"{_BILLING_EXPIRY}.is_team_limited", return_value=True):
_run(adapter)
- adapter.drop_resources.assert_called_once()
+ adapter.drop_stream.assert_called_once()
schema.refresh_from_db()
assert schema.sync_type_config["cdc_broken"]["reason"] == "billing_limit_expired"
@@ -391,6 +393,6 @@ def test_a_failed_billing_check_still_checks_the_slots_lag(team):
with patch(f"{_ACTIVITIES}.blocked_past_buffer_retention", side_effect=RuntimeError("quota cache down")):
_run(adapter)
- adapter.drop_resources.assert_called_once()
+ adapter.drop_stream.assert_called_once()
schema.refresh_from_db()
assert schema.sync_type_config["cdc_broken"]["reason"] == "auto_dropped_critical_lag"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
index cacaabd5063d..f17d1f99aa55 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_extract_activity.py
@@ -42,7 +42,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane
stage_handed_over_reset,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import has_queued_batches
-from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCLag, ChangeEvent
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapter
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.position import PgLSN
from products.warehouse_sources.backend.temporal.data_imports.util import NonRetryableException
@@ -151,7 +151,7 @@ def _stub_app_db_writes():
@contextmanager
def _capture_harness(source, schemas, events=()) -> Iterator[SimpleNamespace]:
- reader = MagicMock(last_commit_end_lsn=None)
+ reader = MagicMock(last_commit_position=None)
reader.read_changes.return_value = iter(events)
reader.truncated_tables = []
# Below CDC_MAX_CHANGES_PER_READ so the bounded read loop treats this as a single drained pass.
@@ -485,7 +485,7 @@ class TestCDCExtractActivity:
schema = _make_schema("users", cdc_mode="streaming", source=source)
mock_get_schemas.return_value = [schema]
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = RuntimeError("connection lost")
mock_reader.truncated_tables = []
mock_adapter = MagicMock()
@@ -595,7 +595,7 @@ class TestErrorClassification:
schema = _make_schema("users", cdc_mode="streaming", source=source)
mock_get_schemas.return_value = [schema]
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = psycopg.errors.InvalidPassword(
'password authentication failed for user "test"'
)
@@ -682,7 +682,7 @@ class TestErrorClassification:
schema = _make_schema("users", cdc_mode="streaming", source=source)
mock_get_schemas.return_value = [schema]
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = exc_cls(exc_message)
mock_reader.truncated_tables = []
mock_adapter = MagicMock()
@@ -749,7 +749,7 @@ class TestErrorClassification:
schema = _make_schema("users", cdc_mode="streaming", source=source)
mock_get_schemas.return_value = [schema]
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_adapter = MagicMock()
mock_adapter.create_reader.return_value = mock_reader
mock_adapter.is_slot_invalidation_error.return_value = False
@@ -767,7 +767,7 @@ class TestErrorClassification:
# Never streamed and never tried to recover — the guard fired first.
mock_reader.connect.assert_not_called()
mock_reader.read_changes.assert_not_called()
- mock_adapter.recreate_slot.assert_not_called()
+ mock_adapter.recreate_stream.assert_not_called()
# Broken state persisted so the schedule stops firing against an unconfigured slot.
mock_mark_broken.assert_called_once()
assert mock_mark_broken.call_args.args[0] is source
@@ -802,7 +802,7 @@ class TestErrorClassification:
schema = _make_schema("users", cdc_mode="streaming", source=source)
mock_get_schemas.return_value = [schema]
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = psycopg.errors.InvalidPassword(
'password authentication failed for user "test"'
)
@@ -850,7 +850,7 @@ class TestErrorClassification:
schema = _make_schema("users", cdc_mode="streaming", source=source)
mock_get_schemas.return_value = [schema]
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = psycopg.OperationalError(
'connection to server at "db" failed: Connection refused'
)
@@ -903,7 +903,7 @@ class TestErrorClassification:
mock_get_schemas.return_value = [schema]
# A non-psycopg error the adapter can't classify falls back to retryable UNKNOWN.
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = RuntimeError("arrow merge blew up")
mock_reader.truncated_tables = []
mock_adapter = MagicMock()
@@ -964,7 +964,7 @@ class TestErrorClassification:
schema = _make_schema("users", cdc_mode="streaming", source=source)
mock_get_schemas.return_value = [schema]
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = psycopg.errors.SyntaxError("invalid syntax")
mock_reader.truncated_tables = []
mock_adapter = MagicMock()
@@ -1002,7 +1002,7 @@ class TestSlotInvalidationRecovery:
"DETAIL: This slot has been invalidated because it exceeded the maximum reserved size."
)
capture.adapter.is_slot_invalidation_error.return_value = True
- capture.adapter.recreate_slot.return_value = {"cdc_consistent_point": "0/AA"}
+ capture.adapter.recreate_stream.return_value = {"cdc_consistent_point": "0/AA"}
with patch(f"{_ACTIVITIES}.clear_slot_loss_markers") as clear_markers:
capture.clear_markers = clear_markers
yield capture
@@ -1014,7 +1014,7 @@ class TestSlotInvalidationRecovery:
with self._invalidated_slot(source, schema) as capture:
- def _recreate_slot(source_arg, tables):
+ def _recreate_stream(source_arg, tables):
# The reset, its buffer purge included, must be complete before the slot is recreated.
# Once the new slot exists, no run hits the invalidation error again, so a reset that
# failed after that point would never repeat and the schema would stream across the gap.
@@ -1023,11 +1023,11 @@ class TestSlotInvalidationRecovery:
capture.clear_markers.assert_not_called()
return {"cdc_consistent_point": "0/AA"}
- capture.adapter.recreate_slot.side_effect = _recreate_slot
+ capture.adapter.recreate_stream.side_effect = _recreate_stream
# Recovery handles the error — the activity must not raise (no pointless Temporal retries).
capture.extract()
- capture.adapter.recreate_slot.assert_called_once_with(source, tables=["public.users"])
+ capture.adapter.recreate_stream.assert_called_once_with(source, tables=["public.users"])
assert source.job_inputs["cdc_consistent_point"] == "0/AA"
source.save.assert_called()
capture.clear_markers.assert_called_once_with(source)
@@ -1053,7 +1053,7 @@ class TestSlotInvalidationRecovery:
with self._invalidated_slot(source, schema) as capture:
capture.extract()
- capture.adapter.recreate_slot.assert_called_once_with(source, tables=["tll.students"])
+ capture.adapter.recreate_stream.assert_called_once_with(source, tables=["tll.students"])
assert source.job_inputs["cdc_consistent_point"] == "0/AA"
source.save.assert_called()
@@ -1062,7 +1062,7 @@ class TestSlotInvalidationRecovery:
schema = _make_schema("users", cdc_mode="streaming", source=source)
with self._invalidated_slot(source, schema) as capture:
- capture.adapter.recreate_slot.side_effect = RuntimeError("cannot recreate slot")
+ capture.adapter.recreate_stream.side_effect = RuntimeError("cannot recreate slot")
capture.activity.info.return_value.attempt = CDC_MAX_EXTRACTION_ATTEMPTS
with pytest.raises(RuntimeError, match="cannot recreate slot"):
capture.extract()
@@ -1090,7 +1090,7 @@ class TestSlotInvalidationRecovery:
patch("products.data_warehouse.backend.facade.api.unpause_external_data_schedule") as unpause,
):
if awaits_slot:
- capture.adapter.recreate_slot.side_effect = RuntimeError("cannot recreate slot")
+ capture.adapter.recreate_stream.side_effect = RuntimeError("cannot recreate slot")
with pytest.raises(RuntimeError, match="cannot recreate slot"):
capture.extract()
else:
@@ -1109,7 +1109,7 @@ class TestSlotInvalidationRecovery:
with pytest.raises(RuntimeError, match="connection lost"):
capture.extract()
- capture.adapter.recreate_slot.assert_not_called()
+ capture.adapter.recreate_stream.assert_not_called()
assert schema.sync_type_config["cdc_mode"] == "streaming"
@@ -1123,8 +1123,8 @@ class TestCleanupOrphanSlotsRetentionCap:
cdc_config = MagicMock()
cdc_config.enabled = True
- cdc_config.slot_name = "posthog_slot"
- cdc_config.publication_name = "posthog_pub"
+ cdc_config.stream_name = "posthog_slot"
+ cdc_config.stream_resources_configured = True
cdc_config.management_mode = "posthog"
cdc_config.auto_drop_slot = True
cdc_config.lag_warning_threshold_mb = 1024
@@ -1132,9 +1132,8 @@ class TestCleanupOrphanSlotsRetentionCap:
mock_adapter = MagicMock()
mock_adapter.parse_cdc_config.return_value = cdc_config
- mock_adapter.get_lag_bytes.return_value = lag_mb * 1024 * 1024
- mock_adapter.get_retention_cap_mb.return_value = cap_mb
- mock_adapter.slot_exists.return_value = False
+ mock_adapter.get_lag.return_value = CDCLag(behind_bytes=lag_mb * 1024 * 1024, retention_cap_mb=cap_mb)
+ mock_adapter.stream_exists.return_value = False
mock_get_adapter.return_value = mock_adapter
return source, mock_adapter
@@ -1165,7 +1164,7 @@ class TestCleanupOrphanSlotsRetentionCap:
cleanup_orphan_slots_activity()
# Dropping + marking broken is the "act" — the broken-state details are covered in test_broken.
- mock_adapter.drop_resources.assert_called_once()
+ mock_adapter.drop_stream.assert_called_once()
mock_mark_broken.assert_called_once()
@patch(
@@ -1192,7 +1191,7 @@ class TestCleanupOrphanSlotsRetentionCap:
cleanup_orphan_slots_activity()
- mock_adapter.drop_resources.assert_not_called()
+ mock_adapter.drop_stream.assert_not_called()
mock_mark_broken.assert_not_called()
@@ -1217,7 +1216,7 @@ class TestFailureVisibilityJobs:
MockSourceModel.objects.get.return_value = source
mock_get_schemas.return_value = schemas
- mock_reader = MagicMock(last_commit_end_lsn=None)
+ mock_reader = MagicMock(last_commit_position=None)
mock_reader.read_changes.side_effect = error
mock_reader.truncated_tables = []
mock_adapter = MagicMock()
@@ -1483,7 +1482,7 @@ class _ScriptedReader:
"""Reader stub that serves preconfigured WAL pages to the bounded read loop.
Each page is (events, rows_consumed, commit_end_lsn). read_changes() exposes that page's
- rows_consumed / last_commit_end_lsn exactly as the real reader does after a peek, so the
+ rows_consumed / last_commit_position exactly as the real reader does after a peek, so the
multi-pass loop sees a full page (rows_consumed >= cap) followed by a drained one (< cap).
"""
@@ -1491,7 +1490,7 @@ class _ScriptedReader:
self._pages = list(pages)
self._idx = 0
self.last_rows_consumed = 0
- self.last_commit_end_lsn = None
+ self.last_commit_position = None
self.truncated_tables: list[str] = []
self.confirmed_positions: list[str] = []
self.on_row_calls = 0
@@ -1524,7 +1523,7 @@ class _ScriptedReader:
events, rows_consumed, commit_end_lsn = self._pages[self._idx]
self._idx += 1
self.last_rows_consumed = rows_consumed
- self.last_commit_end_lsn = commit_end_lsn
+ self.last_commit_position = commit_end_lsn
def gen():
for ev in events:
@@ -1560,7 +1559,7 @@ class TestQuietRunSlotAdvance:
source = _make_source()
act = _make_extract_activity(source)
act.cdc_schemas = [_make_schema("users", source=source)]
- act.reader = MagicMock(last_commit_end_lsn=None)
+ act.reader = MagicMock(last_commit_position=None)
act._pre_read_position = pre_read
act._backlog_drained = drained
act.last_confirmed_lsn = already_confirmed
@@ -1573,7 +1572,7 @@ class TestQuietRunSlotAdvance:
source = _make_source()
act = _make_extract_activity(source)
act.cdc_schemas = [_make_schema("users", source=source)]
- act.reader = MagicMock(last_commit_end_lsn=None)
+ act.reader = MagicMock(last_commit_position=None)
act.reader.confirm_position.side_effect = RuntimeError("slot is active for PID 42")
act._pre_read_position = "0/900"
act._backlog_drained = True
@@ -2000,7 +1999,7 @@ class TestBufferedIngressCapture:
act.schema_by_name = {"users": schema}
act._buffered_table_names = {"users"}
act.batcher = MagicMock(event_count=0)
- act.reader = MagicMock(truncated_tables=["users"], last_commit_end_lsn="0/300")
+ act.reader = MagicMock(truncated_tables=["users"], last_commit_position="0/300")
order = MagicMock()
with (
@@ -2031,7 +2030,7 @@ class TestBufferedIngressCapture:
with _capture_harness(source, [schema], events) as capture:
capture.reader.truncated_tables = [truncated_name]
- capture.reader.last_commit_end_lsn = "0/500"
+ capture.reader.last_commit_position = "0/500"
capture.extract()
capture.buffer.write_batch.assert_not_called()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_metrics.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_metrics.py
index 8afa8e260f87..07976e4cfe47 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_metrics.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_metrics.py
@@ -18,7 +18,7 @@ from products.warehouse_sources.backend.temporal.data_imports.cdc.activities imp
cleanup_orphan_slots_activity,
)
from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import ChangeEventBatcher
-from products.warehouse_sources.backend.temporal.data_imports.cdc.types import ChangeEvent
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCLag, ChangeEvent
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.config import PostgresCDCConfig
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.position import PgLSN
@@ -209,9 +209,10 @@ def _sweeper_adapter(*, lag_bytes=0, retention_cap_mb=None):
yield MagicMock()
adapter.management_connection.side_effect = _conn
- adapter.get_lag_bytes.return_value = lag_bytes
- adapter.get_retention_cap_mb.return_value = retention_cap_mb
- adapter.slot_exists.return_value = False
+ adapter.get_lag.return_value = (
+ None if lag_bytes is None else CDCLag(behind_bytes=lag_bytes, retention_cap_mb=retention_cap_mb)
+ )
+ adapter.stream_exists.return_value = False
return adapter
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/types.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/types.py
index 9fd466990ac6..3c1939f5f471 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/types.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/types.py
@@ -1,13 +1,16 @@
from __future__ import annotations
+import abc
import json
-from collections.abc import Iterable, Iterator, Mapping
+from collections.abc import Callable, Iterable, Iterator, Mapping
from dataclasses import dataclass
from datetime import datetime
from typing import Any, Literal, Protocol, Self
import pyarrow as pa
+from posthog.dataclasses import frozen
+
ManagementMode = Literal["posthog", "self_managed"]
@@ -45,22 +48,42 @@ def decode_job_inputs(job_inputs: Mapping[str, Any] | str | None) -> Mapping[str
@dataclass(frozen=True)
-class CDCConfig:
+class CDCConfig(abc.ABC):
"""Base class for engine-specific CDC configs returned by ``parse_cdc_config``.
- Holds fields that apply to any change-stream engine (slot/publication-style
- identifiers, lag thresholds, management policy). Engine adapters return their
- own subclasses (e.g. ``PostgresCDCConfig``) and add engine-specific fields.
+ Holds fields that apply to any change-stream engine (lag thresholds, management
+ policy). Engine adapters return their own subclasses (e.g. ``PostgresCDCConfig``)
+ and add the engine's stream identifiers.
"""
enabled: bool
- slot_name: str
- publication_name: str
management_mode: ManagementMode
lag_warning_threshold_mb: int
lag_critical_threshold_mb: int
auto_drop_slot: bool
+ @property
+ @abc.abstractmethod
+ def stream_name(self) -> str:
+ """Name of the engine-side change stream capture reads (PG: the replication slot), for logs
+ and messages. Empty when the source stores none, which capture refuses to read."""
+
+ @property
+ @abc.abstractmethod
+ def stream_resources_configured(self) -> bool:
+ """Whether the config names every engine-side resource PostHog may drop (PG: a slot and a
+ publication). The orphan sweeper skips a source without them."""
+
+
+@frozen
+class CDCLag:
+ """How far capture is behind the source's change stream."""
+
+ behind_bytes: int
+ # Engine cap on retained backlog in MB (PG: max_slot_wal_keep_size). None when unlimited or
+ # unknown. Past the cap the engine discards the stream itself, so safety nets act below it.
+ retention_cap_mb: int | None
+
class CDCPosition(Protocol):
"""Opaque replication position. PG=LSN, MySQL=GTID set."""
@@ -112,7 +135,19 @@ class CDCStreamReader(Protocol):
def connect(self) -> None: ...
- def read_changes(self) -> Iterator[ChangeEvent]: ...
+ def read_changes(
+ self, upto_nchanges: int | None = None, on_row: Callable[[], None] | None = None
+ ) -> Iterator[ChangeEvent]:
+ """Yield changes in commit order, stopping at the first transaction boundary at or past
+ ``upto_nchanges`` changes. ``on_row`` runs once per change consumed, including changes that
+ yield no event."""
+ ...
+
+ @property
+ def last_rows_consumed(self) -> int:
+ """Changes consumed by the last ``read_changes`` call. Reaching ``upto_nchanges`` means the
+ stream may hold more."""
+ ...
def confirm_position(self, position: str) -> None: ...
@@ -132,6 +167,9 @@ class CDCStreamReader(Protocol):
def clear_truncated_tables(self) -> None: ...
@property
- def last_commit_end_lsn(self) -> str | None: ...
+ def last_commit_position(self) -> str | None:
+ """Serialized position just past the last transaction read, including one with no row
+ events (a TRUNCATE alone). Confirming it moves the stream past everything read."""
+ ...
def close(self) -> None: ...
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py
index 877ffd68be3d..d037f174fd69 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/neon/source.py
@@ -1,4 +1,5 @@
-from typing import Optional
+from collections.abc import Mapping
+from typing import Any, Optional
from products.warehouse_sources.backend.facade.codebase_detection import detect
from products.warehouse_sources.backend.facade.source_config import (
@@ -104,8 +105,7 @@ class NeonSource(PostgresSource):
config: PostgresSourceConfig,
management_mode: str,
tables: list[str],
- slot_name: str | None = None,
- publication_name: str | None = None,
+ setup: Mapping[str, Any],
require_ssl: bool = True,
team_id: int | None = None,
) -> list[str]:
@@ -117,8 +117,7 @@ class NeonSource(PostgresSource):
config,
management_mode=management_mode,
tables=tables,
- slot_name=slot_name,
- publication_name=publication_name,
+ setup=setup,
require_ssl=require_ssl,
team_id=team_id,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py
index ab4f1b33a1f6..97d5d80b4674 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/neon/test_neon_source.py
@@ -25,7 +25,7 @@ def test_cdc_prerequisites_reject_pooled_host_without_connecting(host):
config = mock.MagicMock(host=host)
with mock.patch.object(PostgresSource, "check_cdc_prerequisites") as super_check:
- errors = NeonSource().check_cdc_prerequisites(config, management_mode="posthog", tables=["users"])
+ errors = NeonSource().check_cdc_prerequisites(config, management_mode="posthog", tables=["users"], setup={})
super_check.assert_not_called()
assert len(errors) == 1
@@ -44,7 +44,9 @@ def test_cdc_prerequisites_delegate_for_direct_hosts(host):
config = mock.MagicMock(host=host)
with mock.patch.object(PostgresSource, "check_cdc_prerequisites", return_value=[]) as super_check:
- errors = NeonSource().check_cdc_prerequisites(config, management_mode="posthog", tables=["users"], team_id=7)
+ errors = NeonSource().check_cdc_prerequisites(
+ config, management_mode="posthog", tables=["users"], setup={}, team_id=7
+ )
super_check.assert_called_once()
assert super_check.call_args.kwargs["team_id"] == 7
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py
index 95527bf4f69d..570f79819e71 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/source.py
@@ -1,4 +1,6 @@
import re
+from collections.abc import Mapping
+from typing import Any
from products.warehouse_sources.backend.facade.codebase_detection import NOT_DETECTABLE
from products.warehouse_sources.backend.facade.source_config import (
@@ -147,8 +149,7 @@ class PlanetScalePostgresSource(PostgresSource):
config: PostgresSourceConfig,
management_mode: str,
tables: list[str],
- slot_name: str | None = None,
- publication_name: str | None = None,
+ setup: Mapping[str, Any],
require_ssl: bool = True,
team_id: int | None = None,
) -> list[str]:
@@ -160,8 +161,7 @@ class PlanetScalePostgresSource(PostgresSource):
config,
management_mode=management_mode,
tables=tables,
- slot_name=slot_name,
- publication_name=publication_name,
+ setup=setup,
require_ssl=require_ssl,
team_id=team_id,
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py
index 3c2011656c6a..682f2a7e6b12 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/planetscale_postgres/test_planetscale_postgres_source.py
@@ -91,7 +91,7 @@ def test_cdc_on_the_psbouncer_port_fails_without_connecting(port):
# logical replication silently could not run over it.
with mock.patch.object(PostgresSource, "check_cdc_prerequisites") as super_check:
errors = PlanetScalePostgresSource().check_cdc_prerequisites(
- _config(port=port), management_mode="managed", tables=["users"]
+ _config(port=port), management_mode="managed", tables=["users"], setup={}
)
super_check.assert_not_called()
@@ -110,7 +110,7 @@ def test_a_port_that_is_not_a_number_is_not_treated_as_psbouncer(port):
def test_cdc_on_the_direct_port_defers_to_postgres():
with mock.patch.object(PostgresSource, "check_cdc_prerequisites", return_value=[]) as super_check:
errors = PlanetScalePostgresSource().check_cdc_prerequisites(
- _config(), management_mode="managed", tables=["users"], team_id=7
+ _config(), management_mode="managed", tables=["users"], setup={}, team_id=7
)
super_check.assert_called_once()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
index 71d8503ef2eb..c017301972f9 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/adapter.py
@@ -13,9 +13,13 @@ from sshtunnel import BaseSSHTunnelForwarderError
from products.warehouse_sources.backend.logic.source_inputs import SourceInputsStore
from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import cdc_error_info
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCLag
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import HostNotAllowedError
from products.warehouse_sources.backend.temporal.data_imports.sources.common.tls import tls_options
-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.config import PostgresCDCConfig
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.config import (
+ PostgresCDCConfig,
+ PostgresSetupNames,
+)
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.errors import (
classify_postgres_cdc_error,
)
@@ -157,34 +161,36 @@ class PostgresCDCAdapter:
management_mode: Literal["posthog", "self_managed"],
tables: list[str],
schema: str,
- slot_name: str | None,
- publication_name: str | None,
+ setup: Mapping[str, Any],
) -> list[str]:
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.prerequisite_validator import (
validate_cdc_prerequisites,
)
+ names = PostgresSetupNames.from_setup(setup)
with self.management_connection(source) as conn:
return validate_cdc_prerequisites(
conn=conn,
management_mode=management_mode,
tables=tables,
schema=schema,
- slot_name=slot_name,
- publication_name=publication_name,
+ slot_name=names.slot_name,
+ publication_name=names.publication_name,
)
- def drop_resources(self, conn: Any, slot_name: str, pub_name: str) -> None:
- drop_slot_and_publication(conn, slot_name, pub_name)
-
- def slot_exists(self, conn: Any, slot_name: str) -> bool:
- return slot_exists(conn, slot_name)
+ def drop_stream(self, conn: Any, source: ExternalDataSource) -> None:
+ cdc_config = self.parse_cdc_config(source)
+ drop_slot_and_publication(conn, cdc_config.slot_name, cdc_config.publication_name)
- def get_lag_bytes(self, conn: Any, slot_name: str) -> int | None:
- return get_slot_lag_bytes(conn, slot_name)
+ def stream_exists(self, conn: Any, source: ExternalDataSource) -> bool:
+ return slot_exists(conn, self.parse_cdc_config(source).slot_name)
- def get_retention_cap_mb(self, conn: Any) -> int | None:
- return get_max_slot_wal_keep_size_mb(conn)
+ def get_lag(self, conn: Any, source: ExternalDataSource) -> CDCLag | None:
+ lag_bytes = get_slot_lag_bytes(conn, self.parse_cdc_config(source).slot_name)
+ retention_cap_mb = get_max_slot_wal_keep_size_mb(conn)
+ if lag_bytes is None:
+ return None
+ return CDCLag(behind_bytes=lag_bytes, retention_cap_mb=retention_cap_mb)
def is_slot_invalidation_error(self, exc: BaseException) -> bool:
return is_slot_invalidation_error(exc)
@@ -204,7 +210,7 @@ class PostgresCDCAdapter:
category = classify_postgres_cdc_error(exc)
return cdc_error_info(category) if category is not None else None
- def recreate_slot(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:
+ def recreate_stream(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:
"""Drop the dead replication slot and create a fresh one against the existing
publication, recreating the publication first when PostHog owns it and it's gone.
@@ -408,6 +414,9 @@ class PostgresCDCAdapter:
source.id,
)
+ def manages_capture_set(self, source: ExternalDataSource) -> bool:
+ return self._managed_publication_name(source) is not None
+
def _alter_publication_membership(self, source: ExternalDataSource, schema: str, table: str, add: bool) -> None:
publication_name = self._managed_publication_name(source)
if publication_name is None:
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/config.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/config.py
index 9025c00ffbfa..7f73df470a52 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/config.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/config.py
@@ -12,6 +12,7 @@ import dataclasses
from collections.abc import Mapping
from typing import TYPE_CHECKING, Any
+from posthog.dataclasses import frozen
from posthog.utils import str_to_bool
from products.warehouse_sources.backend.logic.source_inputs import SourceInputsStore, UnreadableSourceInputsError
@@ -29,16 +30,41 @@ DEFAULT_LAG_WARNING_THRESHOLD_MB = 512
DEFAULT_LAG_CRITICAL_THRESHOLD_MB = 2048
+@frozen
+class PostgresSetupNames:
+ """The slot and publication a CDC setup request names. None where the request leaves one blank."""
+
+ slot_name: str | None
+ publication_name: str | None
+
+ @classmethod
+ def from_setup(cls, setup: Mapping[str, Any]) -> PostgresSetupNames:
+ return cls(
+ slot_name=setup.get("cdc_slot_name") or None,
+ publication_name=setup.get("cdc_publication_name") or None,
+ )
+
+
@dataclasses.dataclass(frozen=True, kw_only=True)
class PostgresCDCConfig(CDCConfig):
"""Typed view of the CDC-related fields stored in ``source.job_inputs``.
- Inherits the universal CDC fields (slot/publication/management/lag) from
- ``CDCConfig`` and adds postgres-specific extras such as ``consistent_point``.
+ Inherits the universal CDC fields (management/lag) from ``CDCConfig`` and adds
+ the replication slot, the publication and the slot's ``consistent_point``.
"""
+ slot_name: str
+ publication_name: str
consistent_point: str | None
+ @property
+ def stream_name(self) -> str:
+ return self.slot_name
+
+ @property
+ def stream_resources_configured(self) -> bool:
+ return bool(self.slot_name and self.publication_name)
+
@classmethod
def from_dict(cls, job_inputs: Mapping[str, Any] | str | None) -> PostgresCDCConfig:
ji = decode_job_inputs(job_inputs)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py
index e1c7263c80e0..9fe31f00ade7 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/stream_reader.py
@@ -370,7 +370,7 @@ class PgCDCStreamReader:
return self._decoder.get_key_columns(table_name)
@property
- def last_commit_end_lsn(self) -> str | None:
+ def last_commit_position(self) -> str | None:
"""End LSN of the most recently committed transaction.
Non-None even when only TRUNCATE messages were decoded (no ChangeEvents).
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py
index 664b1b6108b6..012a00e5070e 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_adapter.py
@@ -7,6 +7,7 @@ import psycopg.errors
from parameterized import parameterized
from sshtunnel import BaseSSHTunnelForwarderError
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCLag
from products.warehouse_sources.backend.temporal.data_imports.sources.common.mixins import HostNotAllowedError
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import (
PostgresCDCAdapter,
@@ -205,7 +206,7 @@ class TestSlotSetupErrorMessage:
assert error == "Failed to create replication slot: connection reset"
-class TestRecreateSlot:
+class TestRecreateStream:
@patch(f"{_ADAPTER}.create_slot_and_publication")
@patch(f"{_ADAPTER}.create_slot", return_value="0/AA")
@patch(f"{_ADAPTER}.publication_exists", return_value=True)
@@ -221,7 +222,7 @@ class TestRecreateSlot:
cdc_publication_name="posthog_pub",
)
- fields = PostgresCDCAdapter().recreate_slot(source, tables=["users", "orders"])
+ fields = PostgresCDCAdapter().recreate_stream(source, tables=["users", "orders"])
assert fields == {"cdc_consistent_point": "0/AA"}
mock_drop.assert_called_once()
@@ -256,7 +257,7 @@ class TestRecreateSlot:
cdc_publication_name="posthog_pub",
)
- fields = PostgresCDCAdapter().recreate_slot(source, tables=tables)
+ fields = PostgresCDCAdapter().recreate_stream(source, tables=tables)
assert fields == {"cdc_consistent_point": "0/BB"}
mock_create_slot_and_pub.assert_called_once()
@@ -280,7 +281,7 @@ class TestRecreateSlot:
)
with pytest.raises(RuntimeError, match="does not exist"):
- PostgresCDCAdapter().recreate_slot(source, tables=["users"])
+ PostgresCDCAdapter().recreate_stream(source, tables=["users"])
mock_create.assert_not_called()
mock_create_slot_and_pub.assert_not_called()
@@ -288,7 +289,7 @@ class TestRecreateSlot:
def test_raises_without_slot_name(self) -> None:
source = _source(cdc_enabled=True)
with pytest.raises(RuntimeError, match="no slot name"):
- PostgresCDCAdapter().recreate_slot(source, tables=[])
+ PostgresCDCAdapter().recreate_stream(source, tables=[])
@patch(f"{_POSTGRES}.time.sleep")
@patch(f"{_ADAPTER}.create_slot")
@@ -311,7 +312,7 @@ class TestRecreateSlot:
cdc_publication_name="posthog_pub",
)
- fields = PostgresCDCAdapter().recreate_slot(source, tables=["users"])
+ fields = PostgresCDCAdapter().recreate_stream(source, tables=["users"])
assert fields == {"cdc_consistent_point": "0/CC"}
assert mock_create_slot.call_count == 2
@@ -341,7 +342,7 @@ class TestRecreateSlot:
cdc_publication_name="posthog_pub",
)
- fields = PostgresCDCAdapter().recreate_slot(source, tables=["users"])
+ fields = PostgresCDCAdapter().recreate_stream(source, tables=["users"])
assert fields == {"cdc_consistent_point": "0/DD"}
assert mock_create_slot.call_count == 2
@@ -426,6 +427,42 @@ class TestGetStatus:
mock_tables.assert_not_called()
+class TestGetLag:
+ @parameterized.expand([("slot_reports_a_position", 4096, 2048), ("slot_missing", None, None)])
+ @patch(f"{_ADAPTER}.get_max_slot_wal_keep_size_mb")
+ @patch(f"{_ADAPTER}.get_slot_lag_bytes")
+ def test_reads_the_lag_and_the_retention_cap_of_the_stored_slot(
+ self, _name, lag_bytes, cap_mb, mock_lag, mock_cap
+ ) -> None:
+ mock_lag.return_value = lag_bytes
+ mock_cap.return_value = cap_mb
+ conn = object()
+
+ lag = PostgresCDCAdapter().get_lag(conn, _source(cdc_enabled=True, cdc_slot_name="slot"))
+
+ mock_lag.assert_called_once_with(conn, "slot")
+ assert lag == (None if lag_bytes is None else CDCLag(behind_bytes=lag_bytes, retention_cap_mb=cap_mb))
+
+
+class TestValidatePrerequisites:
+ @patch(
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.prerequisite_validator.validate_cdc_prerequisites",
+ return_value=[],
+ )
+ @patch(f"{_ADAPTER}.cdc_pg_connection", new_callable=_fake_conn)
+ def test_reads_the_slot_and_publication_from_the_setup_fields(self, _conn, mock_validate) -> None:
+ PostgresCDCAdapter().validate_prerequisites(
+ _source(),
+ management_mode="self_managed",
+ tables=[],
+ schema="public",
+ setup={"cdc_slot_name": "", "cdc_publication_name": "customer_pub"},
+ )
+
+ assert mock_validate.call_args.kwargs["slot_name"] is None
+ assert mock_validate.call_args.kwargs["publication_name"] == "customer_pub"
+
+
class TestIsConnectionError:
# A connect-time timeout to an unreachable source DB is the failure `cdc_status` reports as a
# degraded state; it must classify as a connection error so the endpoint doesn't capture it.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
index 7b8ed6a2c441..32c2c5f370db 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py
@@ -57,6 +57,7 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typ
from products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (
PostgresSourceConfig,
)
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.config import PostgresSetupNames
from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.client_deadline import (
CLIENT_DEADLINE_ERROR,
)
@@ -2201,8 +2202,7 @@ class PostgresSource(
config: PostgresSourceConfig,
management_mode: str,
tables: list[str],
- slot_name: str | None = None,
- publication_name: str | None = None,
+ setup: Mapping[str, Any],
require_ssl: bool = True,
team_id: int | None = None,
) -> list[str]:
@@ -2218,6 +2218,7 @@ class PostgresSource(
_connect_to_postgres,
)
+ names = PostgresSetupNames.from_setup(setup)
with self.with_ssh_tunnel(config, team_id, POSTGRES_HOST_CHECK_SOURCE_TYPE) as (host, port):
conn = _connect_to_postgres(
host=host,
@@ -2238,8 +2239,8 @@ class PostgresSource(
management_mode=management_mode, # type: ignore[arg-type]
tables=tables,
schema=schema,
- slot_name=slot_name,
- publication_name=publication_name,
+ slot_name=names.slot_name,
+ publication_name=names.publication_name,
)
finally:
conn.close()
diff --git a/products/warehouse_sources/backend/tests/api/test_external_data_source.py b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
index 5d90672ee4a6..0e0ab2ebfe9b 100644
--- a/products/warehouse_sources/backend/tests/api/test_external_data_source.py
+++ b/products/warehouse_sources/backend/tests/api/test_external_data_source.py
@@ -11191,7 +11191,7 @@ class TestCheckCDCPrerequisitesForSource(APIBaseTest):
format="json",
)
assert response.status_code == 200, response.content
- assert mock_validate.call_args.kwargs["publication_name"] == "customer_pub"
+ assert mock_validate.call_args.kwargs["setup"]["cdc_publication_name"] == "customer_pub"
assert mock_validate.call_args.kwargs["management_mode"] == "self_managed"
@parameterized.expand(
@@ -11625,7 +11625,7 @@ class TestEnableCDC(APIBaseTest):
assert response.status_code == 200, response.content
# Prereq check forwarded the publication name we received.
- assert mock_check.call_args.kwargs["publication_name"] == "customer_pub"
+ assert mock_check.call_args.kwargs["setup"]["cdc_publication_name"] == "customer_pub"
source.refresh_from_db()
ji = source.job_inputs or {}
@@ -12261,7 +12261,7 @@ class TestRepairCDC(APIBaseTest):
assert "CDC is not enabled" in response.json()["message"]
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot"
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream"
)
def test_repair_cdc_rejects_when_no_active_cdc_schemas(self, mock_recreate) -> None:
source = _make_postgres_source(self.team.pk, self.user, cdc_enabled=True)
@@ -12282,7 +12282,7 @@ class TestRepairCDC(APIBaseTest):
@patch("products.data_warehouse.backend.logic.data_load.service.trigger_external_data_workflow")
@patch("products.data_warehouse.backend.logic.data_load.service.unpause_external_data_schedule")
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot",
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream",
return_value={"cdc_consistent_point": "0/AABBCC"},
)
def test_repair_cdc_resets_schemas_and_resumes_schedules(
@@ -12374,7 +12374,7 @@ class TestRepairCDC(APIBaseTest):
@patch("products.data_warehouse.backend.logic.data_load.service.trigger_external_data_workflow")
@patch("products.data_warehouse.backend.logic.data_load.service.unpause_external_data_schedule")
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot",
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream",
side_effect=psycopg.OperationalError("connection refused"),
)
def test_repair_cdc_failure_keeps_broken_state(
@@ -12405,7 +12405,7 @@ class TestRepairCDC(APIBaseTest):
@patch("products.warehouse_sources.backend.presentation.views.external_data_source.base.capture_exception")
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot",
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream",
side_effect=psycopg.errors.InsufficientPrivilege("must be owner of table orders"),
)
def test_repair_cdc_table_ownership_failure_is_not_captured(self, _mock_recreate, mock_capture) -> None:
@@ -12428,7 +12428,7 @@ class TestRepairCDC(APIBaseTest):
mock_capture.assert_not_called()
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot"
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream"
)
@patch(
"products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.get_status"
@@ -12457,7 +12457,7 @@ class TestRepairCDC(APIBaseTest):
@patch("products.data_warehouse.backend.logic.data_load.service.trigger_external_data_workflow")
@patch("products.data_warehouse.backend.logic.data_load.service.unpause_external_data_schedule")
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot",
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream",
return_value={"cdc_consistent_point": "0/AABBCC"},
)
@patch(
@@ -12491,7 +12491,7 @@ class TestRepairCDC(APIBaseTest):
side_effect=[RuntimeError("temporal down"), None],
)
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot",
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream",
return_value={"cdc_consistent_point": "0/AABBCC"},
)
@patch(
@@ -12531,7 +12531,7 @@ class TestRepairCDC(APIBaseTest):
@patch("products.data_warehouse.backend.logic.data_load.service.trigger_external_data_workflow")
@patch("products.data_warehouse.backend.logic.data_load.service.unpause_external_data_schedule")
@patch(
- "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_slot",
+ "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter.PostgresCDCAdapter.recreate_stream",
return_value={"cdc_consistent_point": "0/AABBCC"},
)
def test_repair_cdc_cancels_running_cdc_jobs(