Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(data-warehouse): add a cdc conformance test kit (#116565)

Daniel RC05adb1bee138
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.py
index e69de29bb2d1..c881fc302478 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.py
@@ -0,0 +1,4 @@
+import pytest
+
+# The suite lives outside a test module, where pytest does not show the two sides of a failed assert.
+pytest.register_assert_rewrite(f"{__name__}.conformance_suite")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
new file mode 100644
index 000000000000..4bcaa1f2fb94
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py
@@ -0,0 +1,644 @@
+"""Harness for the CDC conformance suite.
+
+A scripted source database records committed transactions. An engine renders them in its own change
+stream format, and the kit runs the shared capture and consume code over that stream. Change events
+therefore always come from an engine's reader, never from a fixture typed by hand.
+"""
+
+from __future__ import annotations
+
+import io
+import copy
+import uuid
+import asyncio
+import hashlib
+import datetime as dt
+from collections import defaultdict
+from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Sequence
+from contextlib import AbstractContextManager, ExitStack, asynccontextmanager, contextmanager
+from types import SimpleNamespace, TracebackType
+from typing import Any, Literal, Protocol, Self, cast
+
+from unittest.mock import MagicMock, patch
+
+import pyarrow as pa
+import structlog
+import pyarrow.parquet as pq
+from botocore.exceptions import ClientError
+
+from posthog.dataclasses import frozen
+
+from products.warehouse_sources.backend.models.external_data_schema import CDC_SNAPSHOT_LANE_KEY, ExternalDataSchema
+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+from products.warehouse_sources.backend.temporal.data_imports.cdc.activities import CDCExtractActivity, CDCExtractInput
+from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import CDCSourceAdapter
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (
+ CDC_OP_COLUMN,
+ CDC_SEQ_COLUMN,
+ CDC_TIMESTAMP_COLUMN,
+ DELETED_COLUMN,
+ TOAST_OMITTED_COLUMN,
+ companion_resource_name,
+ enrich_delete_rows,
+ enrich_toast_omitted_rows,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.buffer import get_buffer_prefix
+from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import LanePosition, content_columns
+from products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import resolve_batch
+from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (
+ COMPANION_WRITE_MODE,
+ CONSOLIDATED_WRITE_MODE,
+ CDCSourceManager,
+ ListingProof,
+ build_output_lanes,
+ consolidated_resource_name,
+ purge_buffer_before_handover,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.s3.common import strip_s3_protocol
+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs
+
+_CDC = "products.warehouse_sources.backend.temporal.data_imports.cdc"
+_FACADE = "products.data_warehouse.backend.facade"
+_TEAM_ID = 1
+_FIRST_COMMIT_AT = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)
+
+Key = tuple[Any, ...]
+Row = dict[str, Any]
+# One history row as the suite compares it: the operation, the source columns, and the commit time.
+HistoryEntry = tuple[str, Row, dt.datetime]
+
+
+class SimulatedCrash(Exception):
+ pass
+
+
+@frozen
+class TableDef:
+ # Qualified `schema.table`, as engines name a table in their change stream.
+ name: str
+ columns: dict[str, type]
+ primary_key: tuple[str, ...]
+
+ def key_of(self, row: Row) -> Key:
+ return tuple(row[column] for column in self.primary_key)
+
+
+@frozen
+class Statement:
+ op: Literal["I", "U", "D", "T"]
+ table: TableDef
+ key: Row
+ values: Row
+
+
+def insert(table: TableDef, **row: Any) -> Statement:
+ return Statement(op="I", table=table, key={}, values=row)
+
+
+def update(table: TableDef, key: Row, **changes: Any) -> Statement:
+ return Statement(op="U", table=table, key=key, values=changes)
+
+
+def delete(table: TableDef, **key: Any) -> Statement:
+ return Statement(op="D", table=table, key=key, values={})
+
+
+def truncate(table: TableDef) -> Statement:
+ return Statement(op="T", table=table, key={}, values={})
+
+
+@frozen
+class RowChange:
+ op: Literal["I", "U", "D", "T"]
+ table: str
+ # The row after an insert or update, and the removed row of a delete.
+ row: Row
+ old_row: Row | None
+
+
+@frozen
+class Transaction:
+ # 1-based commit order. Engines derive their own position format from it.
+ number: int
+ committed_at: dt.datetime
+ changes: tuple[RowChange, ...]
+
+
+class SourceDatabase:
+ """The source of truth for a scenario, with the change stream state every engine shares."""
+
+ def __init__(self, tables: Sequence[TableDef]) -> None:
+ self.tables = {table.name: table for table in tables}
+ self.log: list[Transaction] = []
+ # The stream holds every transaction above this number.
+ self.confirmed = 0
+ self.stream_lost = False
+ self.stream_recreations = 0
+ self._rows: dict[str, dict[Key, Row]] = {table.name: {} for table in tables}
+ self._failing_confirms = 0
+
+ @property
+ def head(self) -> int:
+ return len(self.log)
+
+ def rows(self, table: TableDef) -> dict[Key, Row]:
+ return dict(self._rows[table.name])
+
+ def commit(self, *statements: Statement) -> Transaction:
+ changes: list[RowChange] = []
+ for statement in statements:
+ table, rows = statement.table, self._rows[statement.table.name]
+ if statement.op == "I":
+ rows[table.key_of(statement.values)] = dict(statement.values)
+ changes.append(RowChange(op="I", table=table.name, row=dict(statement.values), old_row=None))
+ elif statement.op == "U":
+ old = rows.pop(table.key_of(statement.key))
+ new = {**old, **statement.values}
+ rows[table.key_of(new)] = new
+ changes.append(RowChange(op="U", table=table.name, row=new, old_row=old))
+ elif statement.op == "D":
+ old = rows.pop(table.key_of(statement.key))
+ changes.append(RowChange(op="D", table=table.name, row=old, old_row=old))
+ else:
+ rows.clear()
+ changes.append(RowChange(op="T", table=table.name, row={}, old_row=None))
+ number = self.head + 1
+ transaction = Transaction(
+ number=number, committed_at=_FIRST_COMMIT_AT + dt.timedelta(seconds=number), changes=tuple(changes)
+ )
+ self.log.append(transaction)
+ return transaction
+
+ def primary_keys(self, namespace: str, relations: list[str]) -> dict[str, list[str]]:
+ return {name: list(self.tables[f"{namespace}.{name}"].primary_key) for name in relations}
+
+ def unique_keys(self, namespace: str, relations: list[str]) -> dict[str, list[frozenset[str]]]:
+ return {name: [frozenset(self.tables[f"{namespace}.{name}"].primary_key)] for name in relations}
+
+ def pending(self) -> list[Transaction]:
+ return [transaction for transaction in self.log if transaction.number > self.confirmed]
+
+ def confirm(self, number: int) -> None:
+ if self._failing_confirms:
+ self._failing_confirms -= 1
+ raise SimulatedCrash("the stream position could not be confirmed")
+ self.confirmed = max(self.confirmed, number)
+
+ def fail_next_confirm(self) -> None:
+ self._failing_confirms += 1
+
+ def lose_stream(self) -> None:
+ self.stream_lost = True
+
+ def recreate_stream(self) -> int:
+ """A new stream starts at the current end of the log, so the changes before it are gone."""
+ self.stream_lost = False
+ self.stream_recreations += 1
+ self.confirmed = self.head
+ return self.confirmed
+
+ def expected_history(self, table: TableDef) -> list[HistoryEntry]:
+ """Every row version of the table in commit order, with a key change as a delete and an insert.
+
+ A delete carries the values of the row it removed, which the loader fills in for a delete that
+ arrives with its key only.
+ """
+ expected: list[HistoryEntry] = []
+ for transaction in self.log:
+ at = transaction.committed_at
+ for change in transaction.changes:
+ if change.table != table.name or change.op == "T":
+ continue
+ old = change.old_row
+ if change.op == "U" and old is not None and table.key_of(old) != table.key_of(change.row):
+ expected += [("D", dict(old), at), ("I", dict(change.row), at)]
+ else:
+ expected.append((change.op, dict(change.row), at))
+ return expected
+
+
+class ConformanceEngine(Protocol):
+ """What an engine provides to run the suite: its adapter, and a change stream over the database."""
+
+ source_type: str
+ adapter: CDCSourceAdapter[CDCConfig]
+
+ def job_inputs(self) -> dict[str, Any]: ...
+
+ def installed(self) -> AbstractContextManager[None]:
+ """Connects the engine's own client code to the scripted database for the length of a test."""
+ ...
+
+
+class _AsyncFile:
+ def __init__(self, data: bytes) -> None:
+ self._data = data
+
+ async def __aenter__(self) -> Self:
+ return self
+
+ async def __aexit__(self, *_: object) -> None:
+ return None
+
+ async def read(self) -> bytes:
+ return self._data
+
+
+class MemoryObjectStore:
+ """The parts of the sync and async S3 clients that the change buffer uses."""
+
+ def __init__(self) -> None:
+ self.objects: dict[str, bytes] = {}
+ self._writes = 0
+ self._failing_writes: set[int] = set()
+
+ def fail_write(self, nth: int) -> None:
+ self._failing_writes.add(self._writes + nth)
+
+ @contextmanager
+ def open(self, key: str, mode: str) -> Iterator[io.BytesIO]:
+ if mode == "rb":
+ if key not in self.objects:
+ raise FileNotFoundError(key)
+ yield io.BytesIO(self.objects[key])
+ return
+ self._writes += 1
+ buffer = io.BytesIO()
+ yield buffer
+ written = buffer.getvalue()
+ if self._writes in self._failing_writes:
+ # The S3 client flushes what it buffered when a write fails, so a truncated object lands
+ # under the final name. The writer has to remove it.
+ self.objects[key] = written[: len(written) // 2]
+ raise SimulatedCrash("the buffer file could not be written")
+ self.objects[key] = written
+
+ def rm(self, key: str, recursive: bool = False) -> None:
+ doomed = [k for k in self.objects if k == key or (recursive and k.startswith(f"{key}/"))]
+ if not doomed:
+ raise FileNotFoundError(key)
+ for k in doomed:
+ del self.objects[k]
+
+ def ls(self, prefix: str, detail: bool = False, refresh: bool = False) -> list[str]:
+ keys = sorted(k for k in self.objects if k.startswith(f"{prefix}/"))
+ if not keys:
+ raise FileNotFoundError(prefix)
+ return keys
+
+ def _etag(self, key: str) -> str:
+ return hashlib.md5(self.objects[key]).hexdigest()
+
+ async def _ls(self, prefix: str, detail: bool = True, refresh: bool = False) -> list[dict[str, Any]]:
+ now = dt.datetime.now(tz=dt.UTC)
+ return [
+ {"type": "file", "Key": key, "LastModified": now, "ETag": f'"{self._etag(key)}"'}
+ # The consumer lists by URL, and the store names an object by bucket and path.
+ for key in self.ls(strip_s3_protocol(prefix))
+ ]
+
+ async def open_async(self, key: str, mode: str) -> _AsyncFile:
+ if key not in self.objects:
+ raise FileNotFoundError(key)
+ return _AsyncFile(self.objects[key])
+
+ async def _rm(self, key: str) -> None:
+ self.rm(key)
+
+ def split_path(self, key: str) -> tuple[str, str, None]:
+ bucket, _, rest = key.partition("/")
+ return bucket, rest, None
+
+ def invalidate_cache(self, key: str) -> None:
+ return None
+
+ async def get_s3(self, bucket: str) -> MemoryObjectStore:
+ return self
+
+ async def delete_object(self, *, Bucket: str, Key: str, IfMatch: str) -> None:
+ key = f"{Bucket}/{Key}"
+ if key not in self.objects:
+ raise ClientError({"Error": {"Code": "NoSuchKey"}}, "DeleteObject")
+ if IfMatch.strip('"') != self._etag(key):
+ raise ClientError({"Error": {"Code": "PreconditionFailed"}}, "DeleteObject")
+ del self.objects[key]
+
+
+def _highest_position(rows: Sequence[Row]) -> int | None:
+ positions = [row[CDC_SEQ_COLUMN] for row in rows if row.get(CDC_SEQ_COLUMN) is not None]
+ return max(positions) if positions else None
+
+
+def lane_position(rows: Sequence[Row], key_columns: Sequence[str] | None) -> LanePosition:
+ """What `read_lane_position` returns for a table that holds these rows."""
+ position = _highest_position(rows)
+ if position is None or not key_columns:
+ return LanePosition(position=position, applied={})
+ applied: defaultdict[Key, list[Row]] = defaultdict(list)
+ for row in rows:
+ if row.get(CDC_SEQ_COLUMN) == position:
+ identity = tuple(row.get(column) for column in key_columns)
+ applied[identity].append({column: row[column] for column in content_columns(list(row))})
+ return LanePosition(position=position, applied=dict(applied), key_columns=tuple(key_columns))
+
+
+class Warehouse:
+ """Stands in for the loader and the two Delta tables of one schema.
+
+ It applies a batch with the same shared functions the loader calls, in the same order, and keeps
+ the rows in memory. The Delta writer itself is outside the conformance contract.
+ """
+
+ def __init__(self, table: TableDef) -> None:
+ self._table = table
+ self._keys = list(table.primary_key)
+ self.merged: dict[Key, Row] = {}
+ self.history: list[Row] = []
+
+ def merge(self, batch: pa.Table) -> None:
+ if not batch.num_rows:
+ return
+ existing = self._existing_rows()
+ batch = enrich_toast_omitted_rows(batch, self._keys, existing)
+ batch = enrich_delete_rows(batch, self._keys, existing)
+ batch, _ = resolve_batch(batch, self._keys, cdc_write_mode=CONSOLIDATED_WRITE_MODE)
+ for row in _without_toast_marker(batch).to_pylist():
+ self.merged[self._table.key_of(row)] = row
+
+ def append_history(self, batch: pa.Table) -> None:
+ if not batch.num_rows:
+ return
+ # The loader enriches from the open version of each key. The writer that closes the older
+ # versions is not modelled here, so the open version is the last one appended.
+ current = {self._table.key_of(row): row for row in self.history}
+ existing = self._as_table(list(current.values()))
+ batch = enrich_toast_omitted_rows(batch, self._keys, existing)
+ batch = enrich_delete_rows(batch, self._keys, existing)
+ self.history.extend(_without_toast_marker(batch).to_pylist())
+
+ def reseed(self, rows: dict[Key, Row]) -> None:
+ self.merged = {key: {**row, DELETED_COLUMN: False} for key, row in rows.items()}
+ self.history = []
+
+ def rows(self, *, deleted: bool) -> dict[Key, Row]:
+ return {
+ key: {column: row.get(column) for column in self._table.columns}
+ for key, row in self.merged.items()
+ if bool(row.get(DELETED_COLUMN)) is deleted
+ }
+
+ def _existing_rows(self) -> pa.Table | None:
+ return self._as_table(list(self.merged.values()))
+
+ def _as_table(self, rows: list[Row]) -> pa.Table | None:
+ if not rows:
+ return None
+ return pa.table({column: [row.get(column) for row in rows] for column in self._table.columns})
+
+
+def _without_toast_marker(batch: pa.Table) -> pa.Table:
+ # The loader strips the marker before the write, so no stored row carries it.
+ return batch.drop_columns([TOAST_OMITTED_COLUMN]) if TOAST_OMITTED_COLUMN in batch.column_names else batch
+
+
+class _LaneTableRef:
+ """Stands in for `DeltaTableRef`. The resource name is all the patched position read needs."""
+
+ def __init__(self, resource_name: str, *_: object, **__: object) -> None:
+ self._resource_name = resource_name
+
+ async def get_delta_table(self) -> str:
+ return self._resource_name
+
+
+async def _keep_position_stats(*_: object, **__: object) -> None:
+ return None
+
+
+class ConformanceKit:
+ """Runs capture and consume for one source over an engine, with every store kept in memory."""
+
+ def __init__(self, db: SourceDatabase, engine: ConformanceEngine, captured: Sequence[TableDef]) -> None:
+ self.db = db
+ self.store = MemoryObjectStore()
+ self._engine = engine
+ self._log = structlog.get_logger()
+ # The rows the app database holds. Each run and each assertion gets fresh model instances
+ # built from them, so only a write that reaches the database carries over.
+ self._source_id = uuid.uuid4()
+ self._source_job_inputs = engine.job_inputs()
+ self._schema_rows: dict[str, Row] = {
+ table.name: {
+ "id": uuid.uuid4(),
+ "initial_sync_complete": True,
+ "status": None,
+ "latest_error": None,
+ "sync_type_config": {
+ "cdc_mode": "streaming",
+ "cdc_table_mode": "both",
+ "primary_key_columns": list(table.primary_key),
+ },
+ }
+ for table in captured
+ }
+ self._warehouses = {table.name: Warehouse(table) for table in captured}
+ self._proofs: dict[str, ListingProof] = {}
+ # Every listing stamp of the consume run in progress. The last one is the listing it read.
+ self._listings: list[tuple[dt.datetime, dict[str, str]]] = []
+ # Resource name to the rows of that table, for the consume run in progress.
+ self._lane_rows: dict[str, Callable[[], list[Row]]] = {}
+ self._stack = ExitStack()
+
+ def _load_source(self) -> ExternalDataSource:
+ return ExternalDataSource(
+ id=self._source_id,
+ team_id=_TEAM_ID,
+ source_type=self._engine.source_type,
+ job_inputs=copy.deepcopy(self._source_job_inputs),
+ # A saved row always has one, and adapters read it.
+ created_at=_FIRST_COMMIT_AT,
+ )
+
+ def _load_schema(self, name: str, source: ExternalDataSource) -> ExternalDataSchema:
+ stored = copy.deepcopy(self._schema_rows[name])
+ return ExternalDataSchema(
+ name=name,
+ team_id=_TEAM_ID,
+ source=source,
+ sync_type=ExternalDataSchema.SyncType.CDC,
+ should_sync=True,
+ **stored,
+ )
+
+ def _schema_row(self, schema_id: uuid.UUID | str) -> Row:
+ return next(row for row in self._schema_rows.values() if str(row["id"]) == str(schema_id))
+
+ def __enter__(self) -> Self:
+ kit = self
+
+ def merge_config(
+ schema_id: uuid.UUID | str,
+ team_id: int,
+ *,
+ updates: dict[str, Any] | None = None,
+ removes: Iterable[str] | None = None,
+ mutate: Callable[[dict[str, Any]], None] | None = None,
+ extra_model_fields: dict[str, Any] | None = None,
+ ) -> dict[str, Any]:
+ stored = kit._schema_row(schema_id)
+ config = copy.deepcopy(stored["sync_type_config"])
+ config.update(updates or {})
+ for key in removes or []:
+ config.pop(key, None)
+ if mutate is not None:
+ mutate(config)
+ stored["sync_type_config"] = config
+ for field, value in (extra_model_fields or {}).items():
+ stored[field] = copy.deepcopy(value)
+ return copy.deepcopy(config)
+
+ def save_schema(schema: ExternalDataSchema, *_: object, update_fields: Iterable[str] | None = None) -> None:
+ stored = kit._schema_row(schema.id)
+ for field in update_fields if update_fields is not None else list(stored):
+ if field in stored and field != "id":
+ stored[field] = copy.deepcopy(getattr(schema, field))
+
+ def save_source(source: ExternalDataSource, *_: object, update_fields: Iterable[str] | None = None) -> None:
+ if update_fields is None or "job_inputs" in update_fields:
+ kit._source_job_inputs = copy.deepcopy(source.job_inputs)
+
+ def cdc_schemas(extraction: CDCExtractActivity) -> list[ExternalDataSchema]:
+ assert extraction.source is not None
+ return [kit._load_schema(name, extraction.source) for name in kit._schema_rows]
+
+ async def read_lane_position(resource_name: str, *, key_columns: list[str] | None = None) -> LanePosition:
+ return lane_position(kit._lane_rows[resource_name](), key_columns)
+
+ @asynccontextmanager
+ async def async_client() -> AsyncIterator[MemoryObjectStore]:
+ yield kit.store
+
+ async def stamp_listing(_manager: CDCSourceManager, listed_at: dt.datetime, tail: dict[str, str]) -> None:
+ kit._listings.append((listed_at, dict(tail)))
+
+ source_model = MagicMock()
+ source_model.objects.get.side_effect = lambda **_: kit._load_source()
+ activity = MagicMock()
+ activity.info.return_value = MagicMock(attempt=1)
+
+ patches: list[AbstractContextManager[Any]] = [
+ self._engine.installed(),
+ patch(f"{_CDC}.activities.get_cdc_adapter", return_value=self._engine.adapter),
+ patch(f"{_CDC}.activities.source_type_supports_cdc", return_value=True),
+ patch(f"{_CDC}.activities.ExternalDataSource", source_model),
+ patch.object(CDCExtractActivity, "_get_cdc_schemas", cdc_schemas),
+ patch(f"{_CDC}.activities.update_sync_type_config_keys", merge_config),
+ patch.object(ExternalDataSchema, "save", save_schema),
+ patch.object(ExternalDataSource, "save", save_source),
+ # The lanes read their position from the in-memory tables instead of Delta.
+ patch(f"{_CDC}.source_manager.DeltaTableRef", _LaneTableRef),
+ patch(f"{_CDC}.source_manager.ensure_position_stats", _keep_position_stats),
+ patch(f"{_CDC}.source_manager.read_lane_position", read_lane_position),
+ # Everything below is a boundary: Temporal, Celery, analytics and S3.
+ patch(f"{_CDC}.activities.close_old_connections"),
+ patch(f"{_CDC}.activities.activity", activity),
+ patch(f"{_CDC}.activities.posthoganalytics"),
+ patch(f"{_CDC}.activities.cancel_running_sync", return_value=None),
+ patch(f"{_CDC}.activities.has_queued_batches", return_value=False),
+ patch(f"{_CDC}.activities.clear_slot_loss_markers"),
+ patch(f"{_FACADE}.api.pause_external_data_schedule"),
+ patch(f"{_FACADE}.api.unpause_external_data_schedule"),
+ patch(f"{_FACADE}.api.pause_cdc_extraction_schedule"),
+ patch(f"{_FACADE}.api.emit_sync_alert"),
+ patch(f"{_FACADE}.tasks.schedule_external_data_failure_digest"),
+ patch(f"{_CDC}.buffer.get_s3_client", return_value=self.store),
+ patch(f"{_CDC}.buffer.ensure_bucket"),
+ patch(f"{_CDC}.source_manager.aget_s3_client", async_client),
+ patch.object(CDCSourceManager, "stamp_listing", stamp_listing),
+ ]
+ # A patch that fails to start must not leave the earlier ones active for later tests.
+ with ExitStack() as stack:
+ for entry in patches:
+ stack.enter_context(entry)
+ self._stack = stack.pop_all()
+ return self
+
+ def __exit__(
+ self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None
+ ) -> None:
+ self._stack.close()
+
+ def schema(self, table: TableDef) -> ExternalDataSchema:
+ """The schema as the app database holds it now."""
+ return self._load_schema(table.name, self._load_source())
+
+ def capture(self, *, read_limit: int | None = None) -> None:
+ """One capture run. `read_limit` shrinks the read page, so a small backlog spans several pages."""
+ inputs = CDCExtractInput(team_id=_TEAM_ID, source_id=self._source_id)
+ if read_limit is None:
+ CDCExtractActivity(inputs).run()
+ return
+ with patch(f"{_CDC}.activities.CDC_MAX_CHANGES_PER_READ", read_limit):
+ CDCExtractActivity(inputs).run()
+
+ def consume(self, table: TableDef, *, fail_history_lane: bool = False) -> None:
+ """One scheduled sync of the table. `fail_history_lane` leaves the history table behind the
+ consolidated one, as a run does when its companion job fails."""
+ schema, warehouse = self.schema(table), self._warehouses[table.name]
+ self._lane_rows = {
+ consolidated_resource_name(schema): lambda: list(warehouse.merged.values()),
+ companion_resource_name(schema.name): lambda: warehouse.history,
+ }
+ inputs = SimpleNamespace(team_id=_TEAM_ID, schema_id=str(schema.id), job_id="job")
+
+ async def drain() -> None:
+ lanes, floor = await build_output_lanes(schema, MagicMock(), self._log)
+ manager = CDCSourceManager(
+ cast(SourceInputs, inputs), self._log, deletion_floor=floor, proof=self._proofs.get(table.name)
+ )
+ async for batch in manager.get_items():
+ for lane in lanes:
+ rows = lane.transform(batch) if lane.transform is not None else batch
+ if lane.cdc_write_mode != COMPANION_WRITE_MODE:
+ warehouse.merge(rows)
+ elif not fail_history_lane:
+ warehouse.append_history(rows)
+
+ self._listings.clear()
+ asyncio.run(drain())
+ # Only a run that completed every table proves that it consumed what it listed.
+ if self._listings and not fail_history_lane:
+ listed_at, tail = self._listings[-1]
+ self._proofs[table.name] = ListingProof(listed_at=listed_at, tail=tail)
+
+ def snapshot(self, table: TableDef) -> None:
+ """Completes the table's pending snapshot and hands it over to streaming."""
+ purge_buffer_before_handover(self.schema(table), self._log)
+ self._warehouses[table.name].reseed(self.db.rows(table))
+ stored = self._schema_rows[table.name]
+ config = stored["sync_type_config"]
+ config["cdc_mode"] = "streaming"
+ config.pop("reset_pipeline", None)
+ config.pop(CDC_SNAPSHOT_LANE_KEY, None)
+ stored["initial_sync_complete"] = True
+ self._proofs.pop(table.name, None)
+
+ def live_rows(self, table: TableDef) -> dict[Key, Row]:
+ return self._warehouses[table.name].rows(deleted=False)
+
+ def deleted_rows(self, table: TableDef) -> dict[Key, Row]:
+ return self._warehouses[table.name].rows(deleted=True)
+
+ def history(self, table: TableDef) -> list[HistoryEntry]:
+ return [
+ (row[CDC_OP_COLUMN], {column: row.get(column) for column in table.columns}, row[CDC_TIMESTAMP_COLUMN])
+ for row in self._warehouses[table.name].history
+ ]
+
+ def buffered_rows(self, table: TableDef) -> list[Row]:
+ prefix = strip_s3_protocol(get_buffer_prefix(_TEAM_ID, str(self._schema_rows[table.name]["id"])))
+ rows: list[Row] = []
+ for key in sorted(k for k in self.store.objects if k.startswith(f"{prefix}/")):
+ rows.extend(pq.read_table(io.BytesIO(self.store.objects[key])).to_pylist())
+ return rows
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_suite.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_suite.py
new file mode 100644
index 000000000000..c33bb209d75c
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_suite.py
@@ -0,0 +1,210 @@
+"""The CDC conformance suite. An engine runs it by subclassing it in a test module of its own."""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+
+import pytest
+
+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import CDC_FLUSH_MAX_EVENTS
+from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import CDC_RESET_PENDING_KEY
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (
+ ConformanceEngine,
+ ConformanceKit,
+ SimulatedCrash,
+ SourceDatabase,
+ TableDef,
+ delete,
+ insert,
+ truncate,
+ update,
+)
+
+USERS = TableDef(name="public.users", columns={"id": int, "name": str}, primary_key=("id",))
+ORDERS = TableDef(name="public.orders", columns={"id": int, "status": str}, primary_key=("id",))
+# In the source's change stream, but not a table the source syncs.
+AUDIT = TableDef(name="public.audit", columns={"id": int, "note": str}, primary_key=("id",))
+
+# Above twice the capture flush size, so one transaction spans three buffer files.
+_LARGE_TRANSACTION_ROWS = 2 * CDC_FLUSH_MAX_EVENTS + 2_000
+
+
+class CDCConformanceSuite:
+ def make_engine(self, db: SourceDatabase) -> ConformanceEngine:
+ raise NotImplementedError
+
+ @pytest.fixture
+ def kit(self) -> Iterator[ConformanceKit]:
+ db = SourceDatabase([USERS, ORDERS, AUDIT])
+ with ConformanceKit(db, self.make_engine(db), captured=[USERS, ORDERS]) as kit:
+ yield kit
+
+ def _assert_converged(self, kit: ConformanceKit, table: TableDef) -> None:
+ assert kit.live_rows(table) == kit.db.rows(table)
+ assert kit.history(table) == kit.db.expected_history(table)
+
+ @pytest.mark.parametrize("read_limit", [None, 4], ids=["one_page", "several_pages"])
+ def test_changes_land_in_commit_order(self, kit: ConformanceKit, read_limit: int | None) -> None:
+ db = kit.db
+ db.commit(insert(USERS, id=1, name="a"), insert(ORDERS, id=10, status="new"))
+ db.commit(update(USERS, {"id": 1}, name="b"), update(USERS, {"id": 1}, name="c"))
+ db.commit(insert(AUDIT, id=100, note="ignored"))
+ kit.capture(read_limit=read_limit)
+ db.commit(update(USERS, {"id": 1}, name="d"), insert(USERS, id=2, name="e"), delete(ORDERS, id=10))
+ db.commit(insert(ORDERS, id=11, status="new"))
+ db.commit(insert(AUDIT, id=101, note="ignored"))
+ kit.capture(read_limit=read_limit)
+
+ for table in (USERS, ORDERS):
+ kit.consume(table)
+ self._assert_converged(kit, table)
+ assert db.confirmed == db.head
+
+ def test_stream_with_no_captured_changes_is_still_released(self, kit: ConformanceKit) -> None:
+ kit.db.commit(insert(AUDIT, id=100, note="ignored"))
+
+ kit.capture()
+
+ assert kit.db.confirmed == kit.db.head
+ assert kit.buffered_rows(USERS) == []
+
+ @pytest.mark.parametrize("consume_between", [False, True], ids=["unread_files", "consumed_files"])
+ def test_capture_crash_before_confirm_replays_exactly_once(
+ self, kit: ConformanceKit, consume_between: bool
+ ) -> None:
+ db = kit.db
+ db.commit(insert(USERS, id=1, name="a"))
+ db.commit(update(USERS, {"id": 1}, name="b"), insert(USERS, id=2, name="c"))
+ kit.capture()
+ kit.consume(USERS)
+ db.commit(insert(USERS, id=3, name="e"))
+ # The replayed transaction ends the read, so its rows sit at the position the tables hold.
+ # It changes one key twice, and only the second version matches the consolidated row.
+ db.commit(update(USERS, {"id": 1}, name="d"), update(USERS, {"id": 1}, name="g"), delete(USERS, id=2))
+
+ db.fail_next_confirm()
+ with pytest.raises(SimulatedCrash):
+ kit.capture()
+ if consume_between:
+ kit.consume(USERS)
+ db.commit(insert(USERS, id=4, name="f"))
+ kit.capture()
+ kit.consume(USERS)
+
+ self._assert_converged(kit, USERS)
+
+ def test_consumer_crash_between_tables_replays_exactly_once(self, kit: ConformanceKit) -> None:
+ db = kit.db
+ # Two capture runs write two buffer files, so the first sits wholly below the consolidated
+ # table's position once the history table has fallen behind.
+ db.commit(insert(USERS, id=1, name="a"), insert(USERS, id=2, name="b"))
+ kit.capture()
+ db.commit(update(USERS, {"id": 1}, name="c"))
+ kit.capture()
+
+ kit.consume(USERS, fail_history_lane=True)
+ db.commit(delete(USERS, id=2))
+ kit.capture()
+ kit.consume(USERS)
+
+ self._assert_converged(kit, USERS)
+
+ def test_key_change_removes_the_old_key(self, kit: ConformanceKit) -> None:
+ db = kit.db
+ db.commit(insert(USERS, id=1, name="a"))
+ kit.capture()
+ kit.consume(USERS)
+
+ db.commit(update(USERS, {"id": 1}, id=9, name="b"))
+ kit.capture()
+ kit.consume(USERS)
+
+ self._assert_converged(kit, USERS)
+ assert kit.deleted_rows(USERS) == {(1,): {"id": 1, "name": "a"}}
+
+ @pytest.mark.parametrize("same_run", [True, False], ids=["same_run_as_insert", "later_run"])
+ def test_delete_with_partial_row_keeps_last_values(self, kit: ConformanceKit, same_run: bool) -> None:
+ db = kit.db
+ db.commit(insert(USERS, id=1, name="a"), insert(USERS, id=2, name="b"))
+ if not same_run:
+ kit.capture()
+ kit.consume(USERS)
+
+ db.commit(delete(USERS, id=1))
+ kit.capture()
+ kit.consume(USERS)
+
+ self._assert_converged(kit, USERS)
+ assert kit.deleted_rows(USERS) == {(1,): {"id": 1, "name": "a"}}
+
+ def test_truncate_resets_only_the_truncated_table(self, kit: ConformanceKit) -> None:
+ db = kit.db
+ db.commit(insert(USERS, id=1, name="a"), insert(ORDERS, id=10, status="new"))
+ kit.capture()
+ kit.consume(USERS)
+ kit.consume(ORDERS)
+
+ db.commit(update(USERS, {"id": 1}, name="b"), update(ORDERS, {"id": 10}, status="paid"))
+ db.commit(truncate(USERS))
+ db.commit(insert(USERS, id=2, name="c"))
+ kit.capture()
+
+ users = kit.schema(USERS)
+ assert users.cdc_mode == "snapshot"
+ assert users.sync_type_config["reset_pipeline"] is True
+ assert kit.schema(ORDERS).cdc_mode == "streaming"
+ # The snapshot reads the table after this run, so no change from before it may be replayed.
+ assert kit.buffered_rows(USERS) == []
+ # The stream is past the truncate, so a later run does not reset the table again.
+ assert db.confirmed == db.head
+
+ db.commit(insert(USERS, id=3, name="d"))
+ kit.capture()
+ kit.snapshot(USERS)
+ kit.consume(USERS)
+ kit.consume(ORDERS)
+
+ assert kit.live_rows(USERS) == db.rows(USERS)
+ self._assert_converged(kit, ORDERS)
+
+ @pytest.mark.parametrize("crash", [False, True], ids=["clean_run", "crash_inside_transaction"])
+ def test_large_transaction_lands_exactly_once(self, kit: ConformanceKit, crash: bool) -> None:
+ db = kit.db
+ db.commit(insert(USERS, id=1, name="a"))
+ db.commit(*(insert(USERS, id=i, name=f"n{i}") for i in range(2, 2 + _LARGE_TRANSACTION_ROWS)))
+
+ if crash:
+ kit.store.fail_write(3)
+ with pytest.raises(SimulatedCrash):
+ kit.capture()
+ # Two files of the open transaction are flushed, and the stream must still hold all of it.
+ assert db.confirmed == 1
+ kit.consume(USERS)
+ kit.capture()
+ kit.consume(USERS)
+
+ self._assert_converged(kit, USERS)
+
+ def test_lost_stream_is_recreated_and_tables_resnapshot(self, kit: ConformanceKit) -> None:
+ db = kit.db
+ db.commit(insert(USERS, id=1, name="a"))
+ kit.capture()
+ kit.consume(USERS)
+
+ db.lose_stream()
+ db.commit(update(USERS, {"id": 1}, name="b"))
+ kit.capture()
+
+ users = kit.schema(USERS)
+ assert db.stream_recreations == 1
+ assert users.cdc_mode == "snapshot"
+ assert users.sync_type_config["reset_pipeline"] is True
+ assert CDC_RESET_PENDING_KEY not in users.sync_type_config
+ assert kit.buffered_rows(USERS) == []
+
+ db.commit(insert(USERS, id=2, name="c"))
+ kit.capture()
+ kit.snapshot(USERS)
+ kit.consume(USERS)
+
+ assert kit.live_rows(USERS) == db.rows(USERS)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
new file mode 100644
index 000000000000..a982304c25ee
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py
@@ -0,0 +1,221 @@
+"""An in-memory CDC engine with no Postgres in it.
+
+It implements the adapter and reader interfaces over the scripted database of the conformance kit.
+Its positions are zero-padded transaction numbers, so shared code that assumed an LSN fails here.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable, Iterable, Iterator, Mapping
+from contextlib import contextmanager, nullcontext
+from typing import TYPE_CHECKING, Any, Literal
+
+from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import CDCSourceAdapter
+from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCErrorInfo
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (
+ RowChange,
+ SourceDatabase,
+ Transaction,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig, CDCStreamReader, ChangeEvent
+
+if TYPE_CHECKING:
+ from contextlib import AbstractContextManager
+
+ from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource
+
+_STREAM_NAME = "fake_stream"
+_CAPTURE_SET_NAME = "fake_capture_set"
+# Raised by the adapter methods that only setup, the sweeper and repair call, so a test that starts
+# to reach one fails instead of trusting behavior nothing checks.
+_NOT_EXERCISED = "The conformance suite does not exercise this adapter method"
+
+
+class FakeStreamLostError(Exception):
+ pass
+
+
+def _position(transaction_number: int) -> str:
+ return f"{transaction_number:012d}"
+
+
+class FakeCDCStreamReader:
+ def __init__(self, db: SourceDatabase) -> None:
+ self._db = db
+ self._truncated: list[str] = []
+ self._key_change_columns: dict[str, frozenset[str]] = {}
+ self._last_commit_position: str | None = None
+ self.last_rows_consumed = 0
+
+ def connect(self) -> None:
+ return None
+
+ def read_changes(
+ self, upto_nchanges: int | None = None, on_row: Callable[[], None] | None = None
+ ) -> Iterator[ChangeEvent]:
+ self.last_rows_consumed = 0
+ if self._db.stream_lost:
+ raise FakeStreamLostError("the change stream no longer exists")
+ for transaction in self._db.pending():
+ self._last_commit_position = _position(transaction.number)
+ truncated: list[str] = []
+ for change in transaction.changes:
+ self.last_rows_consumed += 1
+ if on_row is not None:
+ on_row()
+ if change.op == "T":
+ truncated.append(change.table)
+ else:
+ yield self._event(change, transaction)
+ # A truncate becomes visible only after the caller consumed the changes before it.
+ self._truncated.extend(truncated)
+ # A read never splits a transaction, so the limit is checked between transactions.
+ if upto_nchanges is not None and self.last_rows_consumed >= upto_nchanges:
+ return
+
+ def _event(self, change: RowChange, transaction: Transaction) -> ChangeEvent:
+ assert change.op != "T"
+ table = self._db.tables[change.table]
+ previous_values: dict[str, object] | None = None
+ columns = change.row
+ if change.op == "D":
+ # Only the key, as an engine that does not log full row images sends a delete.
+ columns = {column: change.row[column] for column in table.primary_key}
+ elif change.op == "U" and change.old_row is not None:
+ watched = self._key_change_columns.get(change.table, frozenset())
+ changed = {name: change.old_row[name] for name in watched if change.old_row[name] != change.row[name]}
+ previous_values = changed or None
+ return ChangeEvent(
+ operation=change.op,
+ table_name=change.table,
+ position_serialized=_position(transaction.number),
+ timestamp=transaction.committed_at,
+ columns=dict(columns),
+ previous_values=previous_values,
+ )
+
+ def confirm_position(self, position: str) -> None:
+ self._db.confirm(int(position))
+
+ def current_position(self) -> str | None:
+ return _position(self._db.head)
+
+ def get_primary_key_columns(self, schema_name: str, table_names: list[str]) -> dict[str, list[str]]:
+ return self._db.primary_keys(schema_name, table_names)
+
+ def get_enforced_unique_keys(self, schema_name: str, table_names: list[str]) -> dict[str, list[frozenset[str]]]:
+ return self._db.unique_keys(schema_name, table_names)
+
+ def set_key_change_columns(self, columns_by_table: Mapping[str, Iterable[str]]) -> None:
+ self._key_change_columns = {table: frozenset(columns) for table, columns in columns_by_table.items()}
+
+ def get_decoder_key_columns(self, table_name: str) -> list[str]:
+ table = self._db.tables.get(table_name)
+ return list(table.primary_key) if table is not None else []
+
+ @property
+ def truncated_tables(self) -> list[str]:
+ return list(self._truncated)
+
+ def clear_truncated_tables(self) -> None:
+ self._truncated.clear()
+
+ @property
+ def last_commit_end_lsn(self) -> str | None:
+ return self._last_commit_position
+
+ def close(self) -> None:
+ return None
+
+
+class FakeCDCAdapter:
+ def __init__(self, db: SourceDatabase) -> None:
+ self._db = db
+
+ def create_reader(self, source: ExternalDataSource) -> CDCStreamReader:
+ return FakeCDCStreamReader(self._db)
+
+ def position_to_seq(self, position_serialized: str) -> int:
+ return int(position_serialized)
+
+ @contextmanager
+ def management_connection(self, source: ExternalDataSource, connect_timeout: int = 15) -> Iterator[Any]:
+ yield self._db
+
+ def validate_prerequisites(
+ self,
+ source: ExternalDataSource,
+ management_mode: Literal["posthog", "self_managed"],
+ tables: list[str],
+ schema: str,
+ slot_name: str | None,
+ publication_name: str | None,
+ ) -> list[str]:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def drop_resources(self, conn: Any, slot_name: str, pub_name: str) -> None:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def slot_exists(self, conn: Any, slot_name: str) -> bool:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def get_lag_bytes(self, conn: Any, slot_name: str) -> int | None:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def get_retention_cap_mb(self, conn: Any) -> int | None:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def is_slot_invalidation_error(self, exc: BaseException) -> bool:
+ return isinstance(exc, FakeStreamLostError)
+
+ def is_connection_error(self, exc: BaseException) -> bool:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def customer_fixable_error_message(self, exc: BaseException) -> str | None:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def classify_error(self, exc: BaseException) -> CDCErrorInfo | None:
+ return None
+
+ def recreate_slot(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:
+ return {"cdc_consistent_point": _position(self._db.recreate_stream())}
+
+ def parse_cdc_config(self, source: ExternalDataSource) -> CDCConfig:
+ return CDCConfig(
+ enabled=True,
+ slot_name=_STREAM_NAME,
+ publication_name=_CAPTURE_SET_NAME,
+ management_mode="posthog",
+ lag_warning_threshold_mb=512,
+ lag_critical_threshold_mb=2048,
+ auto_drop_slot=True,
+ )
+
+ def setup_resources(self, source: ExternalDataSource, payload: dict[str, Any]) -> tuple[dict[str, Any], str | None]:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def cleanup_resources(self, source: ExternalDataSource) -> None:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def get_status(self, source: ExternalDataSource) -> dict[str, Any]:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def add_table(self, source: ExternalDataSource, schema: str, table: str) -> None:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+ def remove_table(self, source: ExternalDataSource, schema: str, table: str) -> None:
+ raise NotImplementedError(_NOT_EXERCISED)
+
+
+class FakeEngine:
+ source_type = "FakeEngine"
+
+ def __init__(self, db: SourceDatabase) -> None:
+ # Typed as the interface, so that the type checker proves the fake still implements it.
+ self.adapter: CDCSourceAdapter[CDCConfig] = FakeCDCAdapter(db)
+
+ def job_inputs(self) -> dict[str, Any]:
+ return {"schema": "public"}
+
+ def installed(self) -> AbstractContextManager[None]:
+ return nullcontext()
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_fake_engine_conformance.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_fake_engine_conformance.py
new file mode 100644
index 000000000000..148a3ae9db7e
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_fake_engine_conformance.py
@@ -0,0 +1,11 @@
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (
+ ConformanceEngine,
+ SourceDatabase,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_suite import CDCConformanceSuite
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.fake_engine import FakeEngine
+
+
+class TestFakeEngineConformance(CDCConformanceSuite):
+ def make_engine(self, db: SourceDatabase) -> ConformanceEngine:
+ return FakeEngine(db)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/conformance_engine.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/conformance_engine.py
new file mode 100644
index 000000000000..7bc00a72326e
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/conformance_engine.py
@@ -0,0 +1,206 @@
+"""Runs the CDC conformance suite over the Postgres adapter, reader and decoder.
+
+The scripted database is rendered as pgoutput messages and served by a simulated replication slot,
+so every change event comes out of the Postgres reader and decoder themselves. The connection, the
+catalog key lookups and the slot DDL helpers are simulated. The connection answers the three
+statements the reader sends.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Iterator
+from contextlib import contextmanager, nullcontext
+from typing import Any
+
+from unittest.mock import patch
+
+import psycopg.errors
+
+from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import CDCSourceAdapter
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (
+ RowChange,
+ SourceDatabase,
+ TableDef,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapter
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.decoder import _OID_INT8, _OID_TEXT
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.position import PgLSN
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.tests.test_decoder import (
+ _make_begin,
+ _make_commit,
+ _make_delete,
+ _make_insert,
+ _make_relation,
+ _make_truncate,
+ _make_update,
+)
+
+_READER = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.stream_reader"
+_ADAPTER = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter"
+
+_SLOT_NAME = "posthog_slot"
+# WAL bytes between two commits, so that transaction N ends at LSN N * _LSN_STEP.
+_LSN_STEP = 0x100
+
+_PEEK_LIMIT = re.compile(r"pg_logical_slot_peek_binary_changes\('[^']*', NULL, (\d+|NULL),")
+_ADVANCE_TARGET = re.compile(r"pg_replication_slot_advance\('[^']*', '([^']+)'\)")
+
+PeekRow = tuple[str, int, bytes]
+
+
+def _lsn(transaction_number: int) -> str:
+ return PgLSN(transaction_number * _LSN_STEP).serialize()
+
+
+def _text(value: Any) -> tuple[str, str] | None:
+ return None if value is None else ("t", str(value))
+
+
+class _SimulatedSlot:
+ def __init__(self, db: SourceDatabase) -> None:
+ self._db = db
+ self._relation_ids = {name: index + 1 for index, name in enumerate(db.tables)}
+
+ def flush_lsn(self) -> str:
+ return _lsn(self._db.head)
+
+ def advance(self, lsn: str) -> None:
+ target = PgLSN.deserialize(lsn).value // _LSN_STEP
+ if target < self._db.confirmed:
+ raise psycopg.errors.ObjectNotInPrerequisiteState(
+ f"cannot advance replication slot to {lsn}, minimum is {_lsn(self._db.confirmed)}"
+ )
+ self._db.confirm(target)
+
+ def peek(self, limit: int | None) -> Iterator[PeekRow]:
+ if self._db.stream_lost:
+ raise psycopg.errors.ObjectNotInPrerequisiteState(
+ f'can no longer get changes from replication slot "{_SLOT_NAME}"'
+ )
+ # Postgres sends a relation once per decoding session, before the first change that uses it.
+ announced: set[str] = set()
+ emitted = 0
+ for transaction in self._db.pending():
+ end = transaction.number * _LSN_STEP
+ commit = end - 0x10
+ messages = [_make_begin(final_lsn=commit, timestamp=transaction.committed_at, xid=transaction.number)]
+ for change in transaction.changes:
+ table = self._db.tables[change.table]
+ if change.table not in announced:
+ announced.add(change.table)
+ messages.append(self._relation(table))
+ messages.append(self._change(table, change))
+ messages.append(_make_commit(commit_lsn=commit, end_lsn=end, timestamp=transaction.committed_at))
+ for message in messages:
+ emitted += 1
+ yield _lsn(transaction.number), transaction.number, message
+ # A peek never splits a transaction, so the limit is checked after each commit.
+ if limit is not None and emitted >= limit:
+ return
+
+ def _relation(self, table: TableDef) -> bytes:
+ schema, name = table.name.split(".", 1)
+ columns = [(column, _OID_INT8 if kind is int else _OID_TEXT, -1) for column, kind in table.columns.items()]
+ return _make_relation(self._relation_ids[table.name], schema, name, columns, key_columns=set(table.primary_key))
+
+ def _change(self, table: TableDef, change: RowChange) -> bytes:
+ relation_id = self._relation_ids[table.name]
+ if change.op == "T":
+ return _make_truncate([relation_id])
+ new = [_text(change.row.get(column)) for column in table.columns]
+ if change.op == "I":
+ return _make_insert(relation_id, new)
+ assert change.old_row is not None
+ # Under the default replica identity an old tuple holds the key, and NULL in every other column.
+ old_key = [_text(change.old_row[column]) if column in table.primary_key else None for column in table.columns]
+ if change.op == "D":
+ return _make_delete(relation_id, old_key)
+ # Postgres sends the old key of an update only when the update changed it.
+ key_changed = table.key_of(change.old_row) != table.key_of(change.row)
+ return _make_update(relation_id, new, old_values=old_key if key_changed else None)
+
+
+class _SimulatedCursor:
+ itersize = 0
+
+ def __init__(self, slot: _SimulatedSlot) -> None:
+ self._slot = slot
+ self._rows: Iterator[tuple[Any, ...]] = iter(())
+
+ def __enter__(self) -> _SimulatedCursor:
+ return self
+
+ def __exit__(self, *_: object) -> None:
+ return None
+
+ def execute(self, query: Any) -> None:
+ statement = query if isinstance(query, str) else query.as_string(None)
+ if peek := _PEEK_LIMIT.search(statement):
+ self._rows = self._slot.peek(None if peek.group(1) == "NULL" else int(peek.group(1)))
+ elif advance := _ADVANCE_TARGET.search(statement):
+ self._slot.advance(advance.group(1))
+ elif "pg_current_wal_flush_lsn" in statement:
+ self._rows = iter([(self._slot.flush_lsn(),)])
+ else:
+ raise AssertionError(f"The simulated slot does not answer this statement: {statement}")
+
+ def fetchone(self) -> tuple[Any, ...] | None:
+ return next(self._rows, None)
+
+ def __iter__(self) -> Iterator[tuple[Any, ...]]:
+ return self._rows
+
+
+class _SimulatedConnection:
+ def __init__(self, slot: _SimulatedSlot) -> None:
+ self._slot = slot
+
+ def cursor(self, name: str | None = None) -> _SimulatedCursor:
+ return _SimulatedCursor(self._slot)
+
+ def commit(self) -> None:
+ return None
+
+ def rollback(self) -> None:
+ return None
+
+ def close(self) -> None:
+ return None
+
+
+class SimulatedPostgres:
+ source_type = "Postgres"
+
+ def __init__(self, db: SourceDatabase) -> None:
+ self._db = db
+ self._slot = _SimulatedSlot(db)
+ self.adapter: CDCSourceAdapter[CDCConfig] = PostgresCDCAdapter()
+
+ def job_inputs(self) -> dict[str, Any]:
+ return {
+ "host": "db.example.com",
+ "port": 5432,
+ "database": "app",
+ "user": "posthog",
+ "password": "not-a-real-password",
+ "schema": "public",
+ "cdc_enabled": True,
+ "cdc_slot_name": _SLOT_NAME,
+ "cdc_publication_name": "posthog_pub",
+ }
+
+ @contextmanager
+ def installed(self) -> Iterator[None]:
+ with (
+ patch(f"{_READER}._connect_to_postgres", side_effect=lambda **_: _SimulatedConnection(self._slot)),
+ patch(f"{_READER}.get_primary_key_columns", side_effect=lambda _, *args: self._db.primary_keys(*args)),
+ patch(f"{_READER}.get_enforced_unique_keys", side_effect=lambda _, *args: self._db.unique_keys(*args)),
+ # Slot recreation runs DDL over a management connection, which the slot stands in for.
+ patch(f"{_ADAPTER}.cdc_pg_connection", side_effect=lambda *_, **__: nullcontext()),
+ patch(f"{_ADAPTER}.drop_slot"),
+ patch(f"{_ADAPTER}.publication_exists", return_value=True),
+ patch(f"{_ADAPTER}.create_slot", side_effect=lambda *_: _lsn(self._db.recreate_stream())),
+ ):
+ yield
diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_postgres_conformance.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_postgres_conformance.py
new file mode 100644
index 000000000000..bc366c8c1575
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_postgres_conformance.py
@@ -0,0 +1,13 @@
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (
+ ConformanceEngine,
+ SourceDatabase,
+)
+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_suite import CDCConformanceSuite
+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.tests.conformance_engine import (
+ SimulatedPostgres,
+)
+
+
+class TestPostgresConformance(CDCConformanceSuite):
+ def make_engine(self, db: SourceDatabase) -> ConformanceEngine:
+ return SimulatedPostgres(db)