:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(data-warehouse): add a cdc conformance test kit (#116565)
Daniel RC05adb1bee138
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.pyindex e69de29bb2d1..c881fc302478 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/__init__.py@@ -0,0 +1,4 @@+import pytest++# The suite lives outside a test module, where pytest does not show the two sides of a failed assert.+pytest.register_assert_rewrite(f"{__name__}.conformance_suite")diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.pynew file mode 100644index 000000000000..4bcaa1f2fb94--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_kit.py@@ -0,0 +1,644 @@+"""Harness for the CDC conformance suite.++A scripted source database records committed transactions. An engine renders them in its own change+stream format, and the kit runs the shared capture and consume code over that stream. Change events+therefore always come from an engine's reader, never from a fixture typed by hand.+"""++from __future__ import annotations++import io+import copy+import uuid+import asyncio+import hashlib+import datetime as dt+from collections import defaultdict+from collections.abc import AsyncIterator, Callable, Iterable, Iterator, Sequence+from contextlib import AbstractContextManager, ExitStack, asynccontextmanager, contextmanager+from types import SimpleNamespace, TracebackType+from typing import Any, Literal, Protocol, Self, cast++from unittest.mock import MagicMock, patch++import pyarrow as pa+import structlog+import pyarrow.parquet as pq+from botocore.exceptions import ClientError++from posthog.dataclasses import frozen++from products.warehouse_sources.backend.models.external_data_schema import CDC_SNAPSHOT_LANE_KEY, ExternalDataSchema+from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource+from products.warehouse_sources.backend.temporal.data_imports.cdc.activities import CDCExtractActivity, CDCExtractInput+from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import CDCSourceAdapter+from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import (+ CDC_OP_COLUMN,+ CDC_SEQ_COLUMN,+ CDC_TIMESTAMP_COLUMN,+ DELETED_COLUMN,+ TOAST_OMITTED_COLUMN,+ companion_resource_name,+ enrich_delete_rows,+ enrich_toast_omitted_rows,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.buffer import get_buffer_prefix+from products.warehouse_sources.backend.temporal.data_imports.cdc.lane_position import LanePosition, content_columns+from products.warehouse_sources.backend.temporal.data_imports.cdc.load_resolution import resolve_batch+from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (+ COMPANION_WRITE_MODE,+ CONSOLIDATED_WRITE_MODE,+ CDCSourceManager,+ ListingProof,+ build_output_lanes,+ consolidated_resource_name,+ purge_buffer_before_handover,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig+from products.warehouse_sources.backend.temporal.data_imports.pipelines.pipeline_v3.s3.common import strip_s3_protocol+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs++_CDC = "products.warehouse_sources.backend.temporal.data_imports.cdc"+_FACADE = "products.data_warehouse.backend.facade"+_TEAM_ID = 1+_FIRST_COMMIT_AT = dt.datetime(2026, 1, 1, tzinfo=dt.UTC)++Key = tuple[Any, ...]+Row = dict[str, Any]+# One history row as the suite compares it: the operation, the source columns, and the commit time.+HistoryEntry = tuple[str, Row, dt.datetime]+++class SimulatedCrash(Exception):+ pass+++@frozen+class TableDef:+ # Qualified `schema.table`, as engines name a table in their change stream.+ name: str+ columns: dict[str, type]+ primary_key: tuple[str, ...]++ def key_of(self, row: Row) -> Key:+ return tuple(row[column] for column in self.primary_key)+++@frozen+class Statement:+ op: Literal["I", "U", "D", "T"]+ table: TableDef+ key: Row+ values: Row+++def insert(table: TableDef, **row: Any) -> Statement:+ return Statement(op="I", table=table, key={}, values=row)+++def update(table: TableDef, key: Row, **changes: Any) -> Statement:+ return Statement(op="U", table=table, key=key, values=changes)+++def delete(table: TableDef, **key: Any) -> Statement:+ return Statement(op="D", table=table, key=key, values={})+++def truncate(table: TableDef) -> Statement:+ return Statement(op="T", table=table, key={}, values={})+++@frozen+class RowChange:+ op: Literal["I", "U", "D", "T"]+ table: str+ # The row after an insert or update, and the removed row of a delete.+ row: Row+ old_row: Row | None+++@frozen+class Transaction:+ # 1-based commit order. Engines derive their own position format from it.+ number: int+ committed_at: dt.datetime+ changes: tuple[RowChange, ...]+++class SourceDatabase:+ """The source of truth for a scenario, with the change stream state every engine shares."""++ def __init__(self, tables: Sequence[TableDef]) -> None:+ self.tables = {table.name: table for table in tables}+ self.log: list[Transaction] = []+ # The stream holds every transaction above this number.+ self.confirmed = 0+ self.stream_lost = False+ self.stream_recreations = 0+ self._rows: dict[str, dict[Key, Row]] = {table.name: {} for table in tables}+ self._failing_confirms = 0++ @property+ def head(self) -> int:+ return len(self.log)++ def rows(self, table: TableDef) -> dict[Key, Row]:+ return dict(self._rows[table.name])++ def commit(self, *statements: Statement) -> Transaction:+ changes: list[RowChange] = []+ for statement in statements:+ table, rows = statement.table, self._rows[statement.table.name]+ if statement.op == "I":+ rows[table.key_of(statement.values)] = dict(statement.values)+ changes.append(RowChange(op="I", table=table.name, row=dict(statement.values), old_row=None))+ elif statement.op == "U":+ old = rows.pop(table.key_of(statement.key))+ new = {**old, **statement.values}+ rows[table.key_of(new)] = new+ changes.append(RowChange(op="U", table=table.name, row=new, old_row=old))+ elif statement.op == "D":+ old = rows.pop(table.key_of(statement.key))+ changes.append(RowChange(op="D", table=table.name, row=old, old_row=old))+ else:+ rows.clear()+ changes.append(RowChange(op="T", table=table.name, row={}, old_row=None))+ number = self.head + 1+ transaction = Transaction(+ number=number, committed_at=_FIRST_COMMIT_AT + dt.timedelta(seconds=number), changes=tuple(changes)+ )+ self.log.append(transaction)+ return transaction++ def primary_keys(self, namespace: str, relations: list[str]) -> dict[str, list[str]]:+ return {name: list(self.tables[f"{namespace}.{name}"].primary_key) for name in relations}++ def unique_keys(self, namespace: str, relations: list[str]) -> dict[str, list[frozenset[str]]]:+ return {name: [frozenset(self.tables[f"{namespace}.{name}"].primary_key)] for name in relations}++ def pending(self) -> list[Transaction]:+ return [transaction for transaction in self.log if transaction.number > self.confirmed]++ def confirm(self, number: int) -> None:+ if self._failing_confirms:+ self._failing_confirms -= 1+ raise SimulatedCrash("the stream position could not be confirmed")+ self.confirmed = max(self.confirmed, number)++ def fail_next_confirm(self) -> None:+ self._failing_confirms += 1++ def lose_stream(self) -> None:+ self.stream_lost = True++ def recreate_stream(self) -> int:+ """A new stream starts at the current end of the log, so the changes before it are gone."""+ self.stream_lost = False+ self.stream_recreations += 1+ self.confirmed = self.head+ return self.confirmed++ def expected_history(self, table: TableDef) -> list[HistoryEntry]:+ """Every row version of the table in commit order, with a key change as a delete and an insert.++ A delete carries the values of the row it removed, which the loader fills in for a delete that+ arrives with its key only.+ """+ expected: list[HistoryEntry] = []+ for transaction in self.log:+ at = transaction.committed_at+ for change in transaction.changes:+ if change.table != table.name or change.op == "T":+ continue+ old = change.old_row+ if change.op == "U" and old is not None and table.key_of(old) != table.key_of(change.row):+ expected += [("D", dict(old), at), ("I", dict(change.row), at)]+ else:+ expected.append((change.op, dict(change.row), at))+ return expected+++class ConformanceEngine(Protocol):+ """What an engine provides to run the suite: its adapter, and a change stream over the database."""++ source_type: str+ adapter: CDCSourceAdapter[CDCConfig]++ def job_inputs(self) -> dict[str, Any]: ...++ def installed(self) -> AbstractContextManager[None]:+ """Connects the engine's own client code to the scripted database for the length of a test."""+ ...+++class _AsyncFile:+ def __init__(self, data: bytes) -> None:+ self._data = data++ async def __aenter__(self) -> Self:+ return self++ async def __aexit__(self, *_: object) -> None:+ return None++ async def read(self) -> bytes:+ return self._data+++class MemoryObjectStore:+ """The parts of the sync and async S3 clients that the change buffer uses."""++ def __init__(self) -> None:+ self.objects: dict[str, bytes] = {}+ self._writes = 0+ self._failing_writes: set[int] = set()++ def fail_write(self, nth: int) -> None:+ self._failing_writes.add(self._writes + nth)++ @contextmanager+ def open(self, key: str, mode: str) -> Iterator[io.BytesIO]:+ if mode == "rb":+ if key not in self.objects:+ raise FileNotFoundError(key)+ yield io.BytesIO(self.objects[key])+ return+ self._writes += 1+ buffer = io.BytesIO()+ yield buffer+ written = buffer.getvalue()+ if self._writes in self._failing_writes:+ # The S3 client flushes what it buffered when a write fails, so a truncated object lands+ # under the final name. The writer has to remove it.+ self.objects[key] = written[: len(written) // 2]+ raise SimulatedCrash("the buffer file could not be written")+ self.objects[key] = written++ def rm(self, key: str, recursive: bool = False) -> None:+ doomed = [k for k in self.objects if k == key or (recursive and k.startswith(f"{key}/"))]+ if not doomed:+ raise FileNotFoundError(key)+ for k in doomed:+ del self.objects[k]++ def ls(self, prefix: str, detail: bool = False, refresh: bool = False) -> list[str]:+ keys = sorted(k for k in self.objects if k.startswith(f"{prefix}/"))+ if not keys:+ raise FileNotFoundError(prefix)+ return keys++ def _etag(self, key: str) -> str:+ return hashlib.md5(self.objects[key]).hexdigest()++ async def _ls(self, prefix: str, detail: bool = True, refresh: bool = False) -> list[dict[str, Any]]:+ now = dt.datetime.now(tz=dt.UTC)+ return [+ {"type": "file", "Key": key, "LastModified": now, "ETag": f'"{self._etag(key)}"'}+ # The consumer lists by URL, and the store names an object by bucket and path.+ for key in self.ls(strip_s3_protocol(prefix))+ ]++ async def open_async(self, key: str, mode: str) -> _AsyncFile:+ if key not in self.objects:+ raise FileNotFoundError(key)+ return _AsyncFile(self.objects[key])++ async def _rm(self, key: str) -> None:+ self.rm(key)++ def split_path(self, key: str) -> tuple[str, str, None]:+ bucket, _, rest = key.partition("/")+ return bucket, rest, None++ def invalidate_cache(self, key: str) -> None:+ return None++ async def get_s3(self, bucket: str) -> MemoryObjectStore:+ return self++ async def delete_object(self, *, Bucket: str, Key: str, IfMatch: str) -> None:+ key = f"{Bucket}/{Key}"+ if key not in self.objects:+ raise ClientError({"Error": {"Code": "NoSuchKey"}}, "DeleteObject")+ if IfMatch.strip('"') != self._etag(key):+ raise ClientError({"Error": {"Code": "PreconditionFailed"}}, "DeleteObject")+ del self.objects[key]+++def _highest_position(rows: Sequence[Row]) -> int | None:+ positions = [row[CDC_SEQ_COLUMN] for row in rows if row.get(CDC_SEQ_COLUMN) is not None]+ return max(positions) if positions else None+++def lane_position(rows: Sequence[Row], key_columns: Sequence[str] | None) -> LanePosition:+ """What `read_lane_position` returns for a table that holds these rows."""+ position = _highest_position(rows)+ if position is None or not key_columns:+ return LanePosition(position=position, applied={})+ applied: defaultdict[Key, list[Row]] = defaultdict(list)+ for row in rows:+ if row.get(CDC_SEQ_COLUMN) == position:+ identity = tuple(row.get(column) for column in key_columns)+ applied[identity].append({column: row[column] for column in content_columns(list(row))})+ return LanePosition(position=position, applied=dict(applied), key_columns=tuple(key_columns))+++class Warehouse:+ """Stands in for the loader and the two Delta tables of one schema.++ It applies a batch with the same shared functions the loader calls, in the same order, and keeps+ the rows in memory. The Delta writer itself is outside the conformance contract.+ """++ def __init__(self, table: TableDef) -> None:+ self._table = table+ self._keys = list(table.primary_key)+ self.merged: dict[Key, Row] = {}+ self.history: list[Row] = []++ def merge(self, batch: pa.Table) -> None:+ if not batch.num_rows:+ return+ existing = self._existing_rows()+ batch = enrich_toast_omitted_rows(batch, self._keys, existing)+ batch = enrich_delete_rows(batch, self._keys, existing)+ batch, _ = resolve_batch(batch, self._keys, cdc_write_mode=CONSOLIDATED_WRITE_MODE)+ for row in _without_toast_marker(batch).to_pylist():+ self.merged[self._table.key_of(row)] = row++ def append_history(self, batch: pa.Table) -> None:+ if not batch.num_rows:+ return+ # The loader enriches from the open version of each key. The writer that closes the older+ # versions is not modelled here, so the open version is the last one appended.+ current = {self._table.key_of(row): row for row in self.history}+ existing = self._as_table(list(current.values()))+ batch = enrich_toast_omitted_rows(batch, self._keys, existing)+ batch = enrich_delete_rows(batch, self._keys, existing)+ self.history.extend(_without_toast_marker(batch).to_pylist())++ def reseed(self, rows: dict[Key, Row]) -> None:+ self.merged = {key: {**row, DELETED_COLUMN: False} for key, row in rows.items()}+ self.history = []++ def rows(self, *, deleted: bool) -> dict[Key, Row]:+ return {+ key: {column: row.get(column) for column in self._table.columns}+ for key, row in self.merged.items()+ if bool(row.get(DELETED_COLUMN)) is deleted+ }++ def _existing_rows(self) -> pa.Table | None:+ return self._as_table(list(self.merged.values()))++ def _as_table(self, rows: list[Row]) -> pa.Table | None:+ if not rows:+ return None+ return pa.table({column: [row.get(column) for row in rows] for column in self._table.columns})+++def _without_toast_marker(batch: pa.Table) -> pa.Table:+ # The loader strips the marker before the write, so no stored row carries it.+ return batch.drop_columns([TOAST_OMITTED_COLUMN]) if TOAST_OMITTED_COLUMN in batch.column_names else batch+++class _LaneTableRef:+ """Stands in for `DeltaTableRef`. The resource name is all the patched position read needs."""++ def __init__(self, resource_name: str, *_: object, **__: object) -> None:+ self._resource_name = resource_name++ async def get_delta_table(self) -> str:+ return self._resource_name+++async def _keep_position_stats(*_: object, **__: object) -> None:+ return None+++class ConformanceKit:+ """Runs capture and consume for one source over an engine, with every store kept in memory."""++ def __init__(self, db: SourceDatabase, engine: ConformanceEngine, captured: Sequence[TableDef]) -> None:+ self.db = db+ self.store = MemoryObjectStore()+ self._engine = engine+ self._log = structlog.get_logger()+ # The rows the app database holds. Each run and each assertion gets fresh model instances+ # built from them, so only a write that reaches the database carries over.+ self._source_id = uuid.uuid4()+ self._source_job_inputs = engine.job_inputs()+ self._schema_rows: dict[str, Row] = {+ table.name: {+ "id": uuid.uuid4(),+ "initial_sync_complete": True,+ "status": None,+ "latest_error": None,+ "sync_type_config": {+ "cdc_mode": "streaming",+ "cdc_table_mode": "both",+ "primary_key_columns": list(table.primary_key),+ },+ }+ for table in captured+ }+ self._warehouses = {table.name: Warehouse(table) for table in captured}+ self._proofs: dict[str, ListingProof] = {}+ # Every listing stamp of the consume run in progress. The last one is the listing it read.+ self._listings: list[tuple[dt.datetime, dict[str, str]]] = []+ # Resource name to the rows of that table, for the consume run in progress.+ self._lane_rows: dict[str, Callable[[], list[Row]]] = {}+ self._stack = ExitStack()++ def _load_source(self) -> ExternalDataSource:+ return ExternalDataSource(+ id=self._source_id,+ team_id=_TEAM_ID,+ source_type=self._engine.source_type,+ job_inputs=copy.deepcopy(self._source_job_inputs),+ # A saved row always has one, and adapters read it.+ created_at=_FIRST_COMMIT_AT,+ )++ def _load_schema(self, name: str, source: ExternalDataSource) -> ExternalDataSchema:+ stored = copy.deepcopy(self._schema_rows[name])+ return ExternalDataSchema(+ name=name,+ team_id=_TEAM_ID,+ source=source,+ sync_type=ExternalDataSchema.SyncType.CDC,+ should_sync=True,+ **stored,+ )++ def _schema_row(self, schema_id: uuid.UUID | str) -> Row:+ return next(row for row in self._schema_rows.values() if str(row["id"]) == str(schema_id))++ def __enter__(self) -> Self:+ kit = self++ def merge_config(+ schema_id: uuid.UUID | str,+ team_id: int,+ *,+ updates: dict[str, Any] | None = None,+ removes: Iterable[str] | None = None,+ mutate: Callable[[dict[str, Any]], None] | None = None,+ extra_model_fields: dict[str, Any] | None = None,+ ) -> dict[str, Any]:+ stored = kit._schema_row(schema_id)+ config = copy.deepcopy(stored["sync_type_config"])+ config.update(updates or {})+ for key in removes or []:+ config.pop(key, None)+ if mutate is not None:+ mutate(config)+ stored["sync_type_config"] = config+ for field, value in (extra_model_fields or {}).items():+ stored[field] = copy.deepcopy(value)+ return copy.deepcopy(config)++ def save_schema(schema: ExternalDataSchema, *_: object, update_fields: Iterable[str] | None = None) -> None:+ stored = kit._schema_row(schema.id)+ for field in update_fields if update_fields is not None else list(stored):+ if field in stored and field != "id":+ stored[field] = copy.deepcopy(getattr(schema, field))++ def save_source(source: ExternalDataSource, *_: object, update_fields: Iterable[str] | None = None) -> None:+ if update_fields is None or "job_inputs" in update_fields:+ kit._source_job_inputs = copy.deepcopy(source.job_inputs)++ def cdc_schemas(extraction: CDCExtractActivity) -> list[ExternalDataSchema]:+ assert extraction.source is not None+ return [kit._load_schema(name, extraction.source) for name in kit._schema_rows]++ async def read_lane_position(resource_name: str, *, key_columns: list[str] | None = None) -> LanePosition:+ return lane_position(kit._lane_rows[resource_name](), key_columns)++ @asynccontextmanager+ async def async_client() -> AsyncIterator[MemoryObjectStore]:+ yield kit.store++ async def stamp_listing(_manager: CDCSourceManager, listed_at: dt.datetime, tail: dict[str, str]) -> None:+ kit._listings.append((listed_at, dict(tail)))++ source_model = MagicMock()+ source_model.objects.get.side_effect = lambda **_: kit._load_source()+ activity = MagicMock()+ activity.info.return_value = MagicMock(attempt=1)++ patches: list[AbstractContextManager[Any]] = [+ self._engine.installed(),+ patch(f"{_CDC}.activities.get_cdc_adapter", return_value=self._engine.adapter),+ patch(f"{_CDC}.activities.source_type_supports_cdc", return_value=True),+ patch(f"{_CDC}.activities.ExternalDataSource", source_model),+ patch.object(CDCExtractActivity, "_get_cdc_schemas", cdc_schemas),+ patch(f"{_CDC}.activities.update_sync_type_config_keys", merge_config),+ patch.object(ExternalDataSchema, "save", save_schema),+ patch.object(ExternalDataSource, "save", save_source),+ # The lanes read their position from the in-memory tables instead of Delta.+ patch(f"{_CDC}.source_manager.DeltaTableRef", _LaneTableRef),+ patch(f"{_CDC}.source_manager.ensure_position_stats", _keep_position_stats),+ patch(f"{_CDC}.source_manager.read_lane_position", read_lane_position),+ # Everything below is a boundary: Temporal, Celery, analytics and S3.+ patch(f"{_CDC}.activities.close_old_connections"),+ patch(f"{_CDC}.activities.activity", activity),+ patch(f"{_CDC}.activities.posthoganalytics"),+ patch(f"{_CDC}.activities.cancel_running_sync", return_value=None),+ patch(f"{_CDC}.activities.has_queued_batches", return_value=False),+ patch(f"{_CDC}.activities.clear_slot_loss_markers"),+ patch(f"{_FACADE}.api.pause_external_data_schedule"),+ patch(f"{_FACADE}.api.unpause_external_data_schedule"),+ patch(f"{_FACADE}.api.pause_cdc_extraction_schedule"),+ patch(f"{_FACADE}.api.emit_sync_alert"),+ patch(f"{_FACADE}.tasks.schedule_external_data_failure_digest"),+ patch(f"{_CDC}.buffer.get_s3_client", return_value=self.store),+ patch(f"{_CDC}.buffer.ensure_bucket"),+ patch(f"{_CDC}.source_manager.aget_s3_client", async_client),+ patch.object(CDCSourceManager, "stamp_listing", stamp_listing),+ ]+ # A patch that fails to start must not leave the earlier ones active for later tests.+ with ExitStack() as stack:+ for entry in patches:+ stack.enter_context(entry)+ self._stack = stack.pop_all()+ return self++ def __exit__(+ self, exc_type: type[BaseException] | None, exc: BaseException | None, traceback: TracebackType | None+ ) -> None:+ self._stack.close()++ def schema(self, table: TableDef) -> ExternalDataSchema:+ """The schema as the app database holds it now."""+ return self._load_schema(table.name, self._load_source())++ def capture(self, *, read_limit: int | None = None) -> None:+ """One capture run. `read_limit` shrinks the read page, so a small backlog spans several pages."""+ inputs = CDCExtractInput(team_id=_TEAM_ID, source_id=self._source_id)+ if read_limit is None:+ CDCExtractActivity(inputs).run()+ return+ with patch(f"{_CDC}.activities.CDC_MAX_CHANGES_PER_READ", read_limit):+ CDCExtractActivity(inputs).run()++ def consume(self, table: TableDef, *, fail_history_lane: bool = False) -> None:+ """One scheduled sync of the table. `fail_history_lane` leaves the history table behind the+ consolidated one, as a run does when its companion job fails."""+ schema, warehouse = self.schema(table), self._warehouses[table.name]+ self._lane_rows = {+ consolidated_resource_name(schema): lambda: list(warehouse.merged.values()),+ companion_resource_name(schema.name): lambda: warehouse.history,+ }+ inputs = SimpleNamespace(team_id=_TEAM_ID, schema_id=str(schema.id), job_id="job")++ async def drain() -> None:+ lanes, floor = await build_output_lanes(schema, MagicMock(), self._log)+ manager = CDCSourceManager(+ cast(SourceInputs, inputs), self._log, deletion_floor=floor, proof=self._proofs.get(table.name)+ )+ async for batch in manager.get_items():+ for lane in lanes:+ rows = lane.transform(batch) if lane.transform is not None else batch+ if lane.cdc_write_mode != COMPANION_WRITE_MODE:+ warehouse.merge(rows)+ elif not fail_history_lane:+ warehouse.append_history(rows)++ self._listings.clear()+ asyncio.run(drain())+ # Only a run that completed every table proves that it consumed what it listed.+ if self._listings and not fail_history_lane:+ listed_at, tail = self._listings[-1]+ self._proofs[table.name] = ListingProof(listed_at=listed_at, tail=tail)++ def snapshot(self, table: TableDef) -> None:+ """Completes the table's pending snapshot and hands it over to streaming."""+ purge_buffer_before_handover(self.schema(table), self._log)+ self._warehouses[table.name].reseed(self.db.rows(table))+ stored = self._schema_rows[table.name]+ config = stored["sync_type_config"]+ config["cdc_mode"] = "streaming"+ config.pop("reset_pipeline", None)+ config.pop(CDC_SNAPSHOT_LANE_KEY, None)+ stored["initial_sync_complete"] = True+ self._proofs.pop(table.name, None)++ def live_rows(self, table: TableDef) -> dict[Key, Row]:+ return self._warehouses[table.name].rows(deleted=False)++ def deleted_rows(self, table: TableDef) -> dict[Key, Row]:+ return self._warehouses[table.name].rows(deleted=True)++ def history(self, table: TableDef) -> list[HistoryEntry]:+ return [+ (row[CDC_OP_COLUMN], {column: row.get(column) for column in table.columns}, row[CDC_TIMESTAMP_COLUMN])+ for row in self._warehouses[table.name].history+ ]++ def buffered_rows(self, table: TableDef) -> list[Row]:+ prefix = strip_s3_protocol(get_buffer_prefix(_TEAM_ID, str(self._schema_rows[table.name]["id"])))+ rows: list[Row] = []+ for key in sorted(k for k in self.store.objects if k.startswith(f"{prefix}/")):+ rows.extend(pq.read_table(io.BytesIO(self.store.objects[key])).to_pylist())+ return rowsdiff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_suite.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_suite.pynew file mode 100644index 000000000000..c33bb209d75c--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/conformance_suite.py@@ -0,0 +1,210 @@+"""The CDC conformance suite. An engine runs it by subclassing it in a test module of its own."""++from __future__ import annotations++from collections.abc import Iterator++import pytest++from products.warehouse_sources.backend.temporal.data_imports.cdc.batcher import CDC_FLUSH_MAX_EVENTS+from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import CDC_RESET_PENDING_KEY+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (+ ConformanceEngine,+ ConformanceKit,+ SimulatedCrash,+ SourceDatabase,+ TableDef,+ delete,+ insert,+ truncate,+ update,+)++USERS = TableDef(name="public.users", columns={"id": int, "name": str}, primary_key=("id",))+ORDERS = TableDef(name="public.orders", columns={"id": int, "status": str}, primary_key=("id",))+# In the source's change stream, but not a table the source syncs.+AUDIT = TableDef(name="public.audit", columns={"id": int, "note": str}, primary_key=("id",))++# Above twice the capture flush size, so one transaction spans three buffer files.+_LARGE_TRANSACTION_ROWS = 2 * CDC_FLUSH_MAX_EVENTS + 2_000+++class CDCConformanceSuite:+ def make_engine(self, db: SourceDatabase) -> ConformanceEngine:+ raise NotImplementedError++ @pytest.fixture+ def kit(self) -> Iterator[ConformanceKit]:+ db = SourceDatabase([USERS, ORDERS, AUDIT])+ with ConformanceKit(db, self.make_engine(db), captured=[USERS, ORDERS]) as kit:+ yield kit++ def _assert_converged(self, kit: ConformanceKit, table: TableDef) -> None:+ assert kit.live_rows(table) == kit.db.rows(table)+ assert kit.history(table) == kit.db.expected_history(table)++ @pytest.mark.parametrize("read_limit", [None, 4], ids=["one_page", "several_pages"])+ def test_changes_land_in_commit_order(self, kit: ConformanceKit, read_limit: int | None) -> None:+ db = kit.db+ db.commit(insert(USERS, id=1, name="a"), insert(ORDERS, id=10, status="new"))+ db.commit(update(USERS, {"id": 1}, name="b"), update(USERS, {"id": 1}, name="c"))+ db.commit(insert(AUDIT, id=100, note="ignored"))+ kit.capture(read_limit=read_limit)+ db.commit(update(USERS, {"id": 1}, name="d"), insert(USERS, id=2, name="e"), delete(ORDERS, id=10))+ db.commit(insert(ORDERS, id=11, status="new"))+ db.commit(insert(AUDIT, id=101, note="ignored"))+ kit.capture(read_limit=read_limit)++ for table in (USERS, ORDERS):+ kit.consume(table)+ self._assert_converged(kit, table)+ assert db.confirmed == db.head++ def test_stream_with_no_captured_changes_is_still_released(self, kit: ConformanceKit) -> None:+ kit.db.commit(insert(AUDIT, id=100, note="ignored"))++ kit.capture()++ assert kit.db.confirmed == kit.db.head+ assert kit.buffered_rows(USERS) == []++ @pytest.mark.parametrize("consume_between", [False, True], ids=["unread_files", "consumed_files"])+ def test_capture_crash_before_confirm_replays_exactly_once(+ self, kit: ConformanceKit, consume_between: bool+ ) -> None:+ db = kit.db+ db.commit(insert(USERS, id=1, name="a"))+ db.commit(update(USERS, {"id": 1}, name="b"), insert(USERS, id=2, name="c"))+ kit.capture()+ kit.consume(USERS)+ db.commit(insert(USERS, id=3, name="e"))+ # The replayed transaction ends the read, so its rows sit at the position the tables hold.+ # It changes one key twice, and only the second version matches the consolidated row.+ db.commit(update(USERS, {"id": 1}, name="d"), update(USERS, {"id": 1}, name="g"), delete(USERS, id=2))++ db.fail_next_confirm()+ with pytest.raises(SimulatedCrash):+ kit.capture()+ if consume_between:+ kit.consume(USERS)+ db.commit(insert(USERS, id=4, name="f"))+ kit.capture()+ kit.consume(USERS)++ self._assert_converged(kit, USERS)++ def test_consumer_crash_between_tables_replays_exactly_once(self, kit: ConformanceKit) -> None:+ db = kit.db+ # Two capture runs write two buffer files, so the first sits wholly below the consolidated+ # table's position once the history table has fallen behind.+ db.commit(insert(USERS, id=1, name="a"), insert(USERS, id=2, name="b"))+ kit.capture()+ db.commit(update(USERS, {"id": 1}, name="c"))+ kit.capture()++ kit.consume(USERS, fail_history_lane=True)+ db.commit(delete(USERS, id=2))+ kit.capture()+ kit.consume(USERS)++ self._assert_converged(kit, USERS)++ def test_key_change_removes_the_old_key(self, kit: ConformanceKit) -> None:+ db = kit.db+ db.commit(insert(USERS, id=1, name="a"))+ kit.capture()+ kit.consume(USERS)++ db.commit(update(USERS, {"id": 1}, id=9, name="b"))+ kit.capture()+ kit.consume(USERS)++ self._assert_converged(kit, USERS)+ assert kit.deleted_rows(USERS) == {(1,): {"id": 1, "name": "a"}}++ @pytest.mark.parametrize("same_run", [True, False], ids=["same_run_as_insert", "later_run"])+ def test_delete_with_partial_row_keeps_last_values(self, kit: ConformanceKit, same_run: bool) -> None:+ db = kit.db+ db.commit(insert(USERS, id=1, name="a"), insert(USERS, id=2, name="b"))+ if not same_run:+ kit.capture()+ kit.consume(USERS)++ db.commit(delete(USERS, id=1))+ kit.capture()+ kit.consume(USERS)++ self._assert_converged(kit, USERS)+ assert kit.deleted_rows(USERS) == {(1,): {"id": 1, "name": "a"}}++ def test_truncate_resets_only_the_truncated_table(self, kit: ConformanceKit) -> None:+ db = kit.db+ db.commit(insert(USERS, id=1, name="a"), insert(ORDERS, id=10, status="new"))+ kit.capture()+ kit.consume(USERS)+ kit.consume(ORDERS)++ db.commit(update(USERS, {"id": 1}, name="b"), update(ORDERS, {"id": 10}, status="paid"))+ db.commit(truncate(USERS))+ db.commit(insert(USERS, id=2, name="c"))+ kit.capture()++ users = kit.schema(USERS)+ assert users.cdc_mode == "snapshot"+ assert users.sync_type_config["reset_pipeline"] is True+ assert kit.schema(ORDERS).cdc_mode == "streaming"+ # The snapshot reads the table after this run, so no change from before it may be replayed.+ assert kit.buffered_rows(USERS) == []+ # The stream is past the truncate, so a later run does not reset the table again.+ assert db.confirmed == db.head++ db.commit(insert(USERS, id=3, name="d"))+ kit.capture()+ kit.snapshot(USERS)+ kit.consume(USERS)+ kit.consume(ORDERS)++ assert kit.live_rows(USERS) == db.rows(USERS)+ self._assert_converged(kit, ORDERS)++ @pytest.mark.parametrize("crash", [False, True], ids=["clean_run", "crash_inside_transaction"])+ def test_large_transaction_lands_exactly_once(self, kit: ConformanceKit, crash: bool) -> None:+ db = kit.db+ db.commit(insert(USERS, id=1, name="a"))+ db.commit(*(insert(USERS, id=i, name=f"n{i}") for i in range(2, 2 + _LARGE_TRANSACTION_ROWS)))++ if crash:+ kit.store.fail_write(3)+ with pytest.raises(SimulatedCrash):+ kit.capture()+ # Two files of the open transaction are flushed, and the stream must still hold all of it.+ assert db.confirmed == 1+ kit.consume(USERS)+ kit.capture()+ kit.consume(USERS)++ self._assert_converged(kit, USERS)++ def test_lost_stream_is_recreated_and_tables_resnapshot(self, kit: ConformanceKit) -> None:+ db = kit.db+ db.commit(insert(USERS, id=1, name="a"))+ kit.capture()+ kit.consume(USERS)++ db.lose_stream()+ db.commit(update(USERS, {"id": 1}, name="b"))+ kit.capture()++ users = kit.schema(USERS)+ assert db.stream_recreations == 1+ assert users.cdc_mode == "snapshot"+ assert users.sync_type_config["reset_pipeline"] is True+ assert CDC_RESET_PENDING_KEY not in users.sync_type_config+ assert kit.buffered_rows(USERS) == []++ db.commit(insert(USERS, id=2, name="c"))+ kit.capture()+ kit.snapshot(USERS)+ kit.consume(USERS)++ assert kit.live_rows(USERS) == db.rows(USERS)diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.pynew file mode 100644index 000000000000..a982304c25ee--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/fake_engine.py@@ -0,0 +1,221 @@+"""An in-memory CDC engine with no Postgres in it.++It implements the adapter and reader interfaces over the scripted database of the conformance kit.+Its positions are zero-padded transaction numbers, so shared code that assumed an LSN fails here.+"""++from __future__ import annotations++from collections.abc import Callable, Iterable, Iterator, Mapping+from contextlib import contextmanager, nullcontext+from typing import TYPE_CHECKING, Any, Literal++from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import CDCSourceAdapter+from products.warehouse_sources.backend.temporal.data_imports.cdc.errors import CDCErrorInfo+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (+ RowChange,+ SourceDatabase,+ Transaction,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig, CDCStreamReader, ChangeEvent++if TYPE_CHECKING:+ from contextlib import AbstractContextManager++ from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource++_STREAM_NAME = "fake_stream"+_CAPTURE_SET_NAME = "fake_capture_set"+# Raised by the adapter methods that only setup, the sweeper and repair call, so a test that starts+# to reach one fails instead of trusting behavior nothing checks.+_NOT_EXERCISED = "The conformance suite does not exercise this adapter method"+++class FakeStreamLostError(Exception):+ pass+++def _position(transaction_number: int) -> str:+ return f"{transaction_number:012d}"+++class FakeCDCStreamReader:+ def __init__(self, db: SourceDatabase) -> None:+ self._db = db+ self._truncated: list[str] = []+ self._key_change_columns: dict[str, frozenset[str]] = {}+ self._last_commit_position: str | None = None+ self.last_rows_consumed = 0++ def connect(self) -> None:+ return None++ def read_changes(+ self, upto_nchanges: int | None = None, on_row: Callable[[], None] | None = None+ ) -> Iterator[ChangeEvent]:+ self.last_rows_consumed = 0+ if self._db.stream_lost:+ raise FakeStreamLostError("the change stream no longer exists")+ for transaction in self._db.pending():+ self._last_commit_position = _position(transaction.number)+ truncated: list[str] = []+ for change in transaction.changes:+ self.last_rows_consumed += 1+ if on_row is not None:+ on_row()+ if change.op == "T":+ truncated.append(change.table)+ else:+ yield self._event(change, transaction)+ # A truncate becomes visible only after the caller consumed the changes before it.+ self._truncated.extend(truncated)+ # A read never splits a transaction, so the limit is checked between transactions.+ if upto_nchanges is not None and self.last_rows_consumed >= upto_nchanges:+ return++ def _event(self, change: RowChange, transaction: Transaction) -> ChangeEvent:+ assert change.op != "T"+ table = self._db.tables[change.table]+ previous_values: dict[str, object] | None = None+ columns = change.row+ if change.op == "D":+ # Only the key, as an engine that does not log full row images sends a delete.+ columns = {column: change.row[column] for column in table.primary_key}+ elif change.op == "U" and change.old_row is not None:+ watched = self._key_change_columns.get(change.table, frozenset())+ changed = {name: change.old_row[name] for name in watched if change.old_row[name] != change.row[name]}+ previous_values = changed or None+ return ChangeEvent(+ operation=change.op,+ table_name=change.table,+ position_serialized=_position(transaction.number),+ timestamp=transaction.committed_at,+ columns=dict(columns),+ previous_values=previous_values,+ )++ def confirm_position(self, position: str) -> None:+ self._db.confirm(int(position))++ def current_position(self) -> str | None:+ return _position(self._db.head)++ def get_primary_key_columns(self, schema_name: str, table_names: list[str]) -> dict[str, list[str]]:+ return self._db.primary_keys(schema_name, table_names)++ def get_enforced_unique_keys(self, schema_name: str, table_names: list[str]) -> dict[str, list[frozenset[str]]]:+ return self._db.unique_keys(schema_name, table_names)++ def set_key_change_columns(self, columns_by_table: Mapping[str, Iterable[str]]) -> None:+ self._key_change_columns = {table: frozenset(columns) for table, columns in columns_by_table.items()}++ def get_decoder_key_columns(self, table_name: str) -> list[str]:+ table = self._db.tables.get(table_name)+ return list(table.primary_key) if table is not None else []++ @property+ def truncated_tables(self) -> list[str]:+ return list(self._truncated)++ def clear_truncated_tables(self) -> None:+ self._truncated.clear()++ @property+ def last_commit_end_lsn(self) -> str | None:+ return self._last_commit_position++ def close(self) -> None:+ return None+++class FakeCDCAdapter:+ def __init__(self, db: SourceDatabase) -> None:+ self._db = db++ def create_reader(self, source: ExternalDataSource) -> CDCStreamReader:+ return FakeCDCStreamReader(self._db)++ def position_to_seq(self, position_serialized: str) -> int:+ return int(position_serialized)++ @contextmanager+ def management_connection(self, source: ExternalDataSource, connect_timeout: int = 15) -> Iterator[Any]:+ yield self._db++ def validate_prerequisites(+ self,+ source: ExternalDataSource,+ management_mode: Literal["posthog", "self_managed"],+ tables: list[str],+ schema: str,+ slot_name: str | None,+ publication_name: str | None,+ ) -> list[str]:+ raise NotImplementedError(_NOT_EXERCISED)++ def drop_resources(self, conn: Any, slot_name: str, pub_name: str) -> None:+ raise NotImplementedError(_NOT_EXERCISED)++ def slot_exists(self, conn: Any, slot_name: str) -> bool:+ raise NotImplementedError(_NOT_EXERCISED)++ def get_lag_bytes(self, conn: Any, slot_name: str) -> int | None:+ raise NotImplementedError(_NOT_EXERCISED)++ def get_retention_cap_mb(self, conn: Any) -> int | None:+ raise NotImplementedError(_NOT_EXERCISED)++ def is_slot_invalidation_error(self, exc: BaseException) -> bool:+ return isinstance(exc, FakeStreamLostError)++ def is_connection_error(self, exc: BaseException) -> bool:+ raise NotImplementedError(_NOT_EXERCISED)++ def customer_fixable_error_message(self, exc: BaseException) -> str | None:+ raise NotImplementedError(_NOT_EXERCISED)++ def classify_error(self, exc: BaseException) -> CDCErrorInfo | None:+ return None++ def recreate_slot(self, source: ExternalDataSource, tables: list[str]) -> dict[str, Any]:+ return {"cdc_consistent_point": _position(self._db.recreate_stream())}++ def parse_cdc_config(self, source: ExternalDataSource) -> CDCConfig:+ return CDCConfig(+ enabled=True,+ slot_name=_STREAM_NAME,+ publication_name=_CAPTURE_SET_NAME,+ management_mode="posthog",+ lag_warning_threshold_mb=512,+ lag_critical_threshold_mb=2048,+ auto_drop_slot=True,+ )++ def setup_resources(self, source: ExternalDataSource, payload: dict[str, Any]) -> tuple[dict[str, Any], str | None]:+ raise NotImplementedError(_NOT_EXERCISED)++ def cleanup_resources(self, source: ExternalDataSource) -> None:+ raise NotImplementedError(_NOT_EXERCISED)++ def get_status(self, source: ExternalDataSource) -> dict[str, Any]:+ raise NotImplementedError(_NOT_EXERCISED)++ def add_table(self, source: ExternalDataSource, schema: str, table: str) -> None:+ raise NotImplementedError(_NOT_EXERCISED)++ def remove_table(self, source: ExternalDataSource, schema: str, table: str) -> None:+ raise NotImplementedError(_NOT_EXERCISED)+++class FakeEngine:+ source_type = "FakeEngine"++ def __init__(self, db: SourceDatabase) -> None:+ # Typed as the interface, so that the type checker proves the fake still implements it.+ self.adapter: CDCSourceAdapter[CDCConfig] = FakeCDCAdapter(db)++ def job_inputs(self) -> dict[str, Any]:+ return {"schema": "public"}++ def installed(self) -> AbstractContextManager[None]:+ return nullcontext()diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_fake_engine_conformance.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_fake_engine_conformance.pynew file mode 100644index 000000000000..148a3ae9db7e--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/tests/test_fake_engine_conformance.py@@ -0,0 +1,11 @@+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (+ ConformanceEngine,+ SourceDatabase,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_suite import CDCConformanceSuite+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.fake_engine import FakeEngine+++class TestFakeEngineConformance(CDCConformanceSuite):+ def make_engine(self, db: SourceDatabase) -> ConformanceEngine:+ return FakeEngine(db)diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/conformance_engine.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/conformance_engine.pynew file mode 100644index 000000000000..7bc00a72326e--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/conformance_engine.py@@ -0,0 +1,206 @@+"""Runs the CDC conformance suite over the Postgres adapter, reader and decoder.++The scripted database is rendered as pgoutput messages and served by a simulated replication slot,+so every change event comes out of the Postgres reader and decoder themselves. The connection, the+catalog key lookups and the slot DDL helpers are simulated. The connection answers the three+statements the reader sends.+"""++from __future__ import annotations++import re+from collections.abc import Iterator+from contextlib import contextmanager, nullcontext+from typing import Any++from unittest.mock import patch++import psycopg.errors++from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import CDCSourceAdapter+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (+ RowChange,+ SourceDatabase,+ TableDef,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.types import CDCConfig+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter import PostgresCDCAdapter+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.decoder import _OID_INT8, _OID_TEXT+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.position import PgLSN+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.tests.test_decoder import (+ _make_begin,+ _make_commit,+ _make_delete,+ _make_insert,+ _make_relation,+ _make_truncate,+ _make_update,+)++_READER = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.stream_reader"+_ADAPTER = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter"++_SLOT_NAME = "posthog_slot"+# WAL bytes between two commits, so that transaction N ends at LSN N * _LSN_STEP.+_LSN_STEP = 0x100++_PEEK_LIMIT = re.compile(r"pg_logical_slot_peek_binary_changes\('[^']*', NULL, (\d+|NULL),")+_ADVANCE_TARGET = re.compile(r"pg_replication_slot_advance\('[^']*', '([^']+)'\)")++PeekRow = tuple[str, int, bytes]+++def _lsn(transaction_number: int) -> str:+ return PgLSN(transaction_number * _LSN_STEP).serialize()+++def _text(value: Any) -> tuple[str, str] | None:+ return None if value is None else ("t", str(value))+++class _SimulatedSlot:+ def __init__(self, db: SourceDatabase) -> None:+ self._db = db+ self._relation_ids = {name: index + 1 for index, name in enumerate(db.tables)}++ def flush_lsn(self) -> str:+ return _lsn(self._db.head)++ def advance(self, lsn: str) -> None:+ target = PgLSN.deserialize(lsn).value // _LSN_STEP+ if target < self._db.confirmed:+ raise psycopg.errors.ObjectNotInPrerequisiteState(+ f"cannot advance replication slot to {lsn}, minimum is {_lsn(self._db.confirmed)}"+ )+ self._db.confirm(target)++ def peek(self, limit: int | None) -> Iterator[PeekRow]:+ if self._db.stream_lost:+ raise psycopg.errors.ObjectNotInPrerequisiteState(+ f'can no longer get changes from replication slot "{_SLOT_NAME}"'+ )+ # Postgres sends a relation once per decoding session, before the first change that uses it.+ announced: set[str] = set()+ emitted = 0+ for transaction in self._db.pending():+ end = transaction.number * _LSN_STEP+ commit = end - 0x10+ messages = [_make_begin(final_lsn=commit, timestamp=transaction.committed_at, xid=transaction.number)]+ for change in transaction.changes:+ table = self._db.tables[change.table]+ if change.table not in announced:+ announced.add(change.table)+ messages.append(self._relation(table))+ messages.append(self._change(table, change))+ messages.append(_make_commit(commit_lsn=commit, end_lsn=end, timestamp=transaction.committed_at))+ for message in messages:+ emitted += 1+ yield _lsn(transaction.number), transaction.number, message+ # A peek never splits a transaction, so the limit is checked after each commit.+ if limit is not None and emitted >= limit:+ return++ def _relation(self, table: TableDef) -> bytes:+ schema, name = table.name.split(".", 1)+ columns = [(column, _OID_INT8 if kind is int else _OID_TEXT, -1) for column, kind in table.columns.items()]+ return _make_relation(self._relation_ids[table.name], schema, name, columns, key_columns=set(table.primary_key))++ def _change(self, table: TableDef, change: RowChange) -> bytes:+ relation_id = self._relation_ids[table.name]+ if change.op == "T":+ return _make_truncate([relation_id])+ new = [_text(change.row.get(column)) for column in table.columns]+ if change.op == "I":+ return _make_insert(relation_id, new)+ assert change.old_row is not None+ # Under the default replica identity an old tuple holds the key, and NULL in every other column.+ old_key = [_text(change.old_row[column]) if column in table.primary_key else None for column in table.columns]+ if change.op == "D":+ return _make_delete(relation_id, old_key)+ # Postgres sends the old key of an update only when the update changed it.+ key_changed = table.key_of(change.old_row) != table.key_of(change.row)+ return _make_update(relation_id, new, old_values=old_key if key_changed else None)+++class _SimulatedCursor:+ itersize = 0++ def __init__(self, slot: _SimulatedSlot) -> None:+ self._slot = slot+ self._rows: Iterator[tuple[Any, ...]] = iter(())++ def __enter__(self) -> _SimulatedCursor:+ return self++ def __exit__(self, *_: object) -> None:+ return None++ def execute(self, query: Any) -> None:+ statement = query if isinstance(query, str) else query.as_string(None)+ if peek := _PEEK_LIMIT.search(statement):+ self._rows = self._slot.peek(None if peek.group(1) == "NULL" else int(peek.group(1)))+ elif advance := _ADVANCE_TARGET.search(statement):+ self._slot.advance(advance.group(1))+ elif "pg_current_wal_flush_lsn" in statement:+ self._rows = iter([(self._slot.flush_lsn(),)])+ else:+ raise AssertionError(f"The simulated slot does not answer this statement: {statement}")++ def fetchone(self) -> tuple[Any, ...] | None:+ return next(self._rows, None)++ def __iter__(self) -> Iterator[tuple[Any, ...]]:+ return self._rows+++class _SimulatedConnection:+ def __init__(self, slot: _SimulatedSlot) -> None:+ self._slot = slot++ def cursor(self, name: str | None = None) -> _SimulatedCursor:+ return _SimulatedCursor(self._slot)++ def commit(self) -> None:+ return None++ def rollback(self) -> None:+ return None++ def close(self) -> None:+ return None+++class SimulatedPostgres:+ source_type = "Postgres"++ def __init__(self, db: SourceDatabase) -> None:+ self._db = db+ self._slot = _SimulatedSlot(db)+ self.adapter: CDCSourceAdapter[CDCConfig] = PostgresCDCAdapter()++ def job_inputs(self) -> dict[str, Any]:+ return {+ "host": "db.example.com",+ "port": 5432,+ "database": "app",+ "user": "posthog",+ "password": "not-a-real-password",+ "schema": "public",+ "cdc_enabled": True,+ "cdc_slot_name": _SLOT_NAME,+ "cdc_publication_name": "posthog_pub",+ }++ @contextmanager+ def installed(self) -> Iterator[None]:+ with (+ patch(f"{_READER}._connect_to_postgres", side_effect=lambda **_: _SimulatedConnection(self._slot)),+ patch(f"{_READER}.get_primary_key_columns", side_effect=lambda _, *args: self._db.primary_keys(*args)),+ patch(f"{_READER}.get_enforced_unique_keys", side_effect=lambda _, *args: self._db.unique_keys(*args)),+ # Slot recreation runs DDL over a management connection, which the slot stands in for.+ patch(f"{_ADAPTER}.cdc_pg_connection", side_effect=lambda *_, **__: nullcontext()),+ patch(f"{_ADAPTER}.drop_slot"),+ patch(f"{_ADAPTER}.publication_exists", return_value=True),+ patch(f"{_ADAPTER}.create_slot", side_effect=lambda *_: _lsn(self._db.recreate_stream())),+ ):+ yielddiff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_postgres_conformance.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_postgres_conformance.pynew file mode 100644index 000000000000..bc366c8c1575--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/cdc/tests/test_postgres_conformance.py@@ -0,0 +1,13 @@+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_kit import (+ ConformanceEngine,+ SourceDatabase,+)+from products.warehouse_sources.backend.temporal.data_imports.cdc.tests.conformance_suite import CDCConformanceSuite+from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.tests.conformance_engine import (+ SimulatedPostgres,+)+++class TestPostgresConformance(CDCConformanceSuite):+ def make_engine(self, db: SourceDatabase) -> ConformanceEngine:+ return SimulatedPostgres(db)