:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(data-warehouse): lift engine-neutral CDC source code into a mixin (#123573)
Daniel RC9f7aa3ed4e56
diff --git a/products/data_warehouse/backend/models/test/test_external_data_source.py b/products/data_warehouse/backend/models/test/test_external_data_source.pyindex dce04a98d70e..e4ec207192f3 100644--- a/products/data_warehouse/backend/models/test/test_external_data_source.py+++ b/products/data_warehouse/backend/models/test/test_external_data_source.py@@ -1,10 +1,13 @@from posthog.test.base import BaseTest-from unittest.mock import patch+from unittest.mock import MagicMock, patch++from parameterized import parameterizedfrom products.warehouse_sources.backend.facade.models import ExternalDataSourcefrom products.warehouse_sources.backend.facade.types import ExternalDataSourceTypeCLEANUP_PATH = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.source.PostgresSource.cleanup_cdc_resources_on_deletion"+PG_ADAPTER = "products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.adapter"class TestExternalDataSourceSoftDelete(BaseTest):@@ -64,3 +67,31 @@ class TestExternalDataSourceSoftDelete(BaseTest):source.refresh_from_db()self.assertTrue(source.deleted)mock_cleanup.assert_called_once()++ @parameterized.expand([("posthog", True), ("self_managed", False)])+ @patch(f"{PG_ADAPTER}.drop_slot_and_publication")+ @patch(f"{PG_ADAPTER}.cdc_pg_connection")+ @patch("products.data_warehouse.backend.facade.api.delete_cdc_extraction_schedule")+ def test_soft_delete_of_a_cdc_postgres_source_tears_down_its_capture(+ self, management_mode, drops_slot, mock_delete_schedule, mock_connection, mock_drop+ ):+ conn = MagicMock()+ mock_connection.return_value.__enter__.return_value = conn+ source = self._create_source(+ source_type=ExternalDataSourceType.POSTGRES,+ job_inputs={+ "host": "localhost",+ "cdc_enabled": True,+ "cdc_management_mode": management_mode,+ "cdc_slot_name": "posthog_slot",+ "cdc_publication_name": "posthog_pub",+ },+ )++ source.soft_delete()++ mock_delete_schedule.assert_called_once_with(str(source.id))+ if drops_slot:+ mock_drop.assert_called_once_with(conn, "posthog_slot", "posthog_pub")+ else:+ mock_drop.assert_not_called()diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.pynew file mode 100644index 000000000000..f6a0ece85011--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/cdc/source_mixin.py@@ -0,0 +1,184 @@+"""The part of a CDC-capable source that does not depend on its database engine."""++from __future__ import annotations++import logging+from typing import TYPE_CHECKING++from products.warehouse_sources.backend.temporal.data_imports.cdc.adapters import (+ get_cdc_adapter,+ source_type_supports_cdc,+)+from products.warehouse_sources.backend.temporal.data_imports.sources.common.base import ConfigType, _BaseSource+from products.warehouse_sources.backend.temporal.data_imports.sources.common.typings import SourceInputs, SourceResponse++if TYPE_CHECKING:+ from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchema+ from products.warehouse_sources.backend.models.external_data_source import ExternalDataSource++log = logging.getLogger(__name__)+++class CDCSourceMixin(_BaseSource[ConfigType]):+ """Change data capture for a source whose engine has an adapter in `cdc/adapters.py`.++ The engine specifics stay behind the adapter, so a new engine implements only+ `check_cdc_prerequisites` here. Subclassing `_BaseSource` puts these hooks ahead of its no-op+ defaults in every source's method resolution order, whatever the order of the bases.+ """++ def check_cdc_prerequisites(+ self,+ config: ConfigType,+ management_mode: str,+ tables: list[str],+ slot_name: str | None = None,+ publication_name: str | None = None,+ require_ssl: bool = True,+ team_id: int | None = None,+ ) -> list[str]:+ """Validate the CDC prerequisites against a live connection, before the source exists.++ Returns one message per unmet prerequisite.+ """+ raise NotImplementedError(f"{type(self).__name__} does not check CDC prerequisites")++ def cleanup_cdc_resources_on_deletion(self, source: ExternalDataSource) -> None:+ """Drop the Temporal schedule + PostHog-managed slot/publication.++ Schedule lives on our side, slot lives on the customer's DB. No-op for+ sources without CDC enabled.+ """+ # A source type with no adapter can never have enabled CDC.+ if not source_type_supports_cdc(source.source_type):+ return+ adapter = get_cdc_adapter(source)+ cdc_config = adapter.parse_cdc_config(source)+ if not cdc_config.enabled:+ return++ # Lazy: data_load.service pulls in Temporal client / Celery setup we don't want at module load.+ from products.data_warehouse.backend.facade.api import delete_cdc_extraction_schedule++ # Schedule key = source id. NotFound is a no-op.+ try:+ delete_cdc_extraction_schedule(str(source.id))+ except Exception:+ log.exception("Failed to delete CDC extraction schedule", extra={"source_id": str(source.id)})++ if cdc_config.management_mode != "posthog":+ return+ if not cdc_config.slot_name or not cdc_config.publication_name:+ return++ try:+ with adapter.management_connection(source, connect_timeout=10) as conn:+ adapter.drop_resources(conn, cdc_config.slot_name, cdc_config.publication_name)+ except Exception:+ log.exception(+ "Failed to drop CDC slot/publication on source DB (best-effort)",+ extra={"source_id": str(source.id), "slot_name": cdc_config.slot_name},+ )++ def _buffered_cdc_source(self, schema: ExternalDataSchema, inputs: SourceInputs) -> SourceResponse:+ """A `SourceResponse` reading this streaming, seeded schema's S3 change buffer."""+ from asgiref.sync import async_to_sync++ from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob+ from products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobs import (+ retire_orphaned_companions,+ )+ from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import hand_reset_to_capture+ from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (+ CDCSourceManager,+ buffer_expired_unread,+ build_output_lanes,+ clear_listing,+ completed_listing_proof,+ has_batches_in_flight,+ served_lanes,+ )++ if not served_lanes(schema):+ raise ValueError(+ f"CDC schema {schema.name} has cdc_table_mode {schema.cdc_table_mode!r}, which no buffer lane "+ "writes. Set it to 'consolidated', 'cdc_only' or 'both'."+ )++ def no_op_tick(*, reset_handed_over: bool = False) -> SourceResponse:+ # An empty response no-ops this tick and keeps the schedule alive. Nothing is listed and+ # nothing is deleted. An earlier attempt of this same job may have stamped a listing, though, and+ # the workflow completes the job on this response — so the stamp comes off, or a batch+ # of that attempt failing later would leave a Completed job proving a listing nothing+ # drained.+ clear_listing(inputs.job_id, inputs.team_id)+ first_lane = served_lanes(schema)[0]+ return SourceResponse(+ name=first_lane.resource_name,+ items=lambda: iter(()),+ primary_keys=schema.primary_key_columns,+ cdc_write_mode=first_lane.write_mode,+ # A change stream carries no seekable key, and `supports_resume` defaults to True.+ supports_resume=False,+ reset_handed_over=reset_handed_over,+ )++ job = ExternalDataJob.objects.filter(id=inputs.job_id, team_id=inputs.team_id).first()++ # A CDC reset must travel through snapshot mode, which re-seeds the table before the buffer+ # replays over it. Merging the buffer into a wiped table instead would leave only the rows+ # changed since. A reset that reaches a streaming table anyway goes to capture, which puts the+ # table back into snapshot mode. Refusing it would fail every run of the table until a manual reset.+ # This run must not delete the table itself: capture waits for queued batches to land first.+ if inputs.reset_pipeline:+ inputs.logger.warning("cdc_buffered_reset_handed_to_capture", schema_name=schema.name)+ hand_reset_to_capture(schema, inputs.logger, start_capture=False)+ return no_op_tick(reset_handed_over=True)++ if has_batches_in_flight(schema):+ # Reading now would stage rows alongside a delivery that is still landing: a previous+ # attempt of this job holds staged batches that are still claimable, which the append+ # lane would then write twice.+ inputs.logger.info("cdc_buffered_waiting_for_in_flight_batches", schema_name=schema.name)+ return no_op_tick()++ if job is None:+ raise ValueError(f"Buffered CDC schema {schema.name} has no job row for run {inputs.job_id}")++ proof = async_to_sync(completed_listing_proof)(schema)+ # The bucket deletes a buffer file once it is older than BUFFER_FILE_RETENTION. A table that has+ # consumed nothing for longer may have lost changes it never loaded, so reading on would leave it+ # wrong for good, and only a re-snapshot makes it correct. Capture does the reset once this run+ # has finished, as it does for any reset a sync could interfere with. A recent proof settles it+ # without the longer read.+ if proof is None and async_to_sync(buffer_expired_unread)(schema):+ inputs.logger.warning(+ "cdc_buffer_expired_before_consumption", schema_name=schema.name, last_synced_at=schema.last_synced_at+ )+ hand_reset_to_capture(schema, inputs.logger, start_capture=False)+ return no_op_tick()++ # Nothing of any earlier run is executing now, so a companion still Running belongs to a+ # run that died without its `finally` and nothing else will ever close it.+ retired = retire_orphaned_companions(schema)+ if retired:+ inputs.logger.warning("cdc_orphaned_companion_jobs_retired", schema_name=schema.name, job_ids=retired)++ # Every table this schema's changes feed, written from one read of the buffer. Each lane+ # carries where its own table stops, because a failed run can leave one ahead of the other.+ lanes, deletion_floor = async_to_sync(build_output_lanes)(schema, job, inputs.logger)+ manager = CDCSourceManager(+ inputs,+ inputs.logger,+ deletion_floor=deletion_floor,+ proof=proof,+ )+ return SourceResponse(+ name=lanes[0].name,+ items=manager.get_items,+ primary_keys=schema.primary_key_columns,+ cdc_write_mode=lanes[0].cdc_write_mode,+ lanes=lanes,+ # A change stream reads a buffer, not a keyed table, so there is no key to seek past.+ supports_resume=False,+ )diff --git a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.pyindex bc5616ffe8d3..7b8ed6a2c441 100644--- a/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py+++ b/products/warehouse_sources/backend/temporal/data_imports/sources/postgres/source.py@@ -1,4 +1,3 @@-import loggingfrom collections.abc import Mappingfrom typing import TYPE_CHECKING, Any, Optional, cast@@ -10,7 +9,6 @@ from sshtunnel import BaseSSHTunnelForwarderErrorfrom products.warehouse_sources.backend.facade.codebase_detection import Detection, NotDetectable, detectif TYPE_CHECKING:- from products.warehouse_sources.backend.models.external_data_schema import ExternalDataSchemafrom products.warehouse_sources.backend.models.external_data_source import ExternalDataSourcefrom posthog.exceptions_capture import capture_exception@@ -24,6 +22,7 @@ from products.warehouse_sources.backend.facade.source_config import (SourceFieldInputConfigType,SourceFieldSSHTunnelConfig,)+from products.warehouse_sources.backend.temporal.data_imports.cdc.source_mixin import CDCSourceMixinfrom products.warehouse_sources.backend.temporal.data_imports.naming_convention import NamingConventionfrom products.warehouse_sources.backend.temporal.data_imports.sources.common.base import (FAST_RETURN_PROBE_TIMEOUT,@@ -58,11 +57,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.common.typfrom products.warehouse_sources.backend.temporal.data_imports.sources.generated_configs.postgres import (PostgresSourceConfig,)-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.config import PostgresCDCConfig-from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.cdc.slot_manager import (- cdc_pg_connection,- drop_slot_and_publication,-)from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.client_deadline import (CLIENT_DEADLINE_ERROR,)@@ -96,8 +90,6 @@ from products.warehouse_sources.backend.temporal.data_imports.sources.postgres.x)from products.warehouse_sources.backend.types import ExternalDataSourceType, IncrementalField, IncrementalFieldType-log = logging.getLogger(__name__)-_HOST_IS_URL_ERROR = ("Enter just the hostname in the host field (for example, db.example.com), not a full URL or ""connection string. Remove any scheme (like http:// or postgres://) and any username, "@@ -571,6 +563,7 @@ _POOLER_AUTH_CIRCUIT_BREAKER_EXHAUSTED_MESSAGE = (@SourceRegistry.registerclass PostgresSource(+ CDCSourceMixin[PostgresSourceConfig],SQLSource[PostgresSourceConfig],ResumableSource[PostgresSourceConfig, KeysetResumeState],CursorSource[XminCursor],@@ -1848,39 +1841,6 @@ class PostgresSource("""Delegates to `reconcile_postgres_schemas` so direct-query mode also rebuilds DWH tables."""return reconcile_postgres_schemas(source=source, source_schemas=source_schemas, team_id=team_id)- def cleanup_cdc_resources_on_deletion(self, source: "ExternalDataSource") -> None:- """Drop the Temporal schedule + PostHog-managed slot/publication.-- Schedule lives on our side, slot lives on the customer's DB. No-op for- postgres sources without CDC enabled.- """- cdc_config = PostgresCDCConfig.from_source(source)- if not cdc_config.enabled:- return-- # Lazy: data_load.service pulls in Temporal client / Celery setup we don't want at module load.- from products.data_warehouse.backend.facade.api import delete_cdc_extraction_schedule-- # Schedule key = source id. NotFound is a no-op.- try:- delete_cdc_extraction_schedule(str(source.id))- except Exception:- log.exception("Failed to delete CDC extraction schedule", extra={"source_id": str(source.id)})-- if cdc_config.management_mode != "posthog":- return- if not cdc_config.slot_name or not cdc_config.publication_name:- return-- try:- with cdc_pg_connection(source, connect_timeout=10) as conn:- drop_slot_and_publication(conn, cdc_config.slot_name, cdc_config.publication_name)- except Exception:- log.exception(- "Failed to drop CDC slot/publication on source DB (best-effort)",- extra={"source_id": str(source.id), "slot_name": cdc_config.slot_name},- )-def get_schemas(self,config: PostgresSourceConfig,@@ -2341,109 +2301,6 @@ class PostgresSource(inputs.logger.debug(f"probe_new_data: falling back to a full sync: {e}", exc_info=e)return None- def _buffered_cdc_source(self, schema: "ExternalDataSchema", inputs: SourceInputs) -> SourceResponse:- """A `SourceResponse` reading this streaming, seeded schema's S3 change buffer."""- from asgiref.sync import async_to_sync-- from products.warehouse_sources.backend.models.external_data_job import ExternalDataJob- from products.warehouse_sources.backend.temporal.data_imports.cdc.companion_jobs import (- retire_orphaned_companions,- )- from products.warehouse_sources.backend.temporal.data_imports.cdc.snapshot_lane import hand_reset_to_capture- from products.warehouse_sources.backend.temporal.data_imports.cdc.source_manager import (- CDCSourceManager,- buffer_expired_unread,- build_output_lanes,- clear_listing,- completed_listing_proof,- has_batches_in_flight,- served_lanes,- )-- if not served_lanes(schema):- raise ValueError(- f"CDC schema {schema.name} has cdc_table_mode {schema.cdc_table_mode!r}, which no buffer lane "- "writes. Set it to 'consolidated', 'cdc_only' or 'both'."- )-- def no_op_tick(*, reset_handed_over: bool = False) -> SourceResponse:- # An empty response no-ops this tick and keeps the schedule alive. Nothing is listed and- # nothing is deleted. An earlier attempt of this same job may have stamped a listing, though, and- # the workflow completes the job on this response — so the stamp comes off, or a batch- # of that attempt failing later would leave a Completed job proving a listing nothing- # drained.- clear_listing(inputs.job_id, inputs.team_id)- first_lane = served_lanes(schema)[0]- return SourceResponse(- name=first_lane.resource_name,- items=lambda: iter(()),- primary_keys=schema.primary_key_columns,- cdc_write_mode=first_lane.write_mode,- # A change stream carries no seekable key, and `supports_resume` defaults to True.- supports_resume=False,- reset_handed_over=reset_handed_over,- )-- job = ExternalDataJob.objects.filter(id=inputs.job_id, team_id=inputs.team_id).first()-- # A CDC reset must travel through snapshot mode, which re-seeds the table before the buffer- # replays over it. Merging the buffer into a wiped table instead would leave only the rows- # changed since. A reset that reaches a streaming table anyway goes to capture, which puts the- # table back into snapshot mode. Refusing it would fail every run of the table until a manual reset.- # This run must not delete the table itself: capture waits for queued batches to land first.- if inputs.reset_pipeline:- inputs.logger.warning("cdc_buffered_reset_handed_to_capture", schema_name=schema.name)- hand_reset_to_capture(schema, inputs.logger, start_capture=False)- return no_op_tick(reset_handed_over=True)-- if has_batches_in_flight(schema):- # Reading now would stage rows alongside a delivery that is still landing: a previous- # attempt of this job holds staged batches that are still claimable, which the append- # lane would then write twice.- inputs.logger.info("cdc_buffered_waiting_for_in_flight_batches", schema_name=schema.name)- return no_op_tick()-- if job is None:- raise ValueError(f"Buffered CDC schema {schema.name} has no job row for run {inputs.job_id}")-- proof = async_to_sync(completed_listing_proof)(schema)- # The bucket deletes a buffer file once it is older than BUFFER_FILE_RETENTION. A table that has- # consumed nothing for longer may have lost changes it never loaded, so reading on would leave it- # wrong for good, and only a re-snapshot makes it correct. Capture does the reset once this run- # has finished, as it does for any reset a sync could interfere with. A recent proof settles it- # without the longer read.- if proof is None and async_to_sync(buffer_expired_unread)(schema):- inputs.logger.warning(- "cdc_buffer_expired_before_consumption", schema_name=schema.name, last_synced_at=schema.last_synced_at- )- hand_reset_to_capture(schema, inputs.logger, start_capture=False)- return no_op_tick()-- # Nothing of any earlier run is executing now, so a companion still Running belongs to a- # run that died without its `finally` and nothing else will ever close it.- retired = retire_orphaned_companions(schema)- if retired:- inputs.logger.warning("cdc_orphaned_companion_jobs_retired", schema_name=schema.name, job_ids=retired)-- # Every table this schema's changes feed, written from one read of the buffer. Each lane- # carries where its own table stops, because a failed run can leave one ahead of the other.- lanes, deletion_floor = async_to_sync(build_output_lanes)(schema, job, inputs.logger)- manager = CDCSourceManager(- inputs,- inputs.logger,- deletion_floor=deletion_floor,- proof=proof,- )- return SourceResponse(- name=lanes[0].name,- items=manager.get_items,- primary_keys=schema.primary_key_columns,- cdc_write_mode=lanes[0].cdc_write_mode,- lanes=lanes,- # A change stream reads a buffer, not a keyed table, so there is no key to seek past.- supports_resume=False,- )-def source_for_pipeline( # type: ignore[override]self,config: PostgresSourceConfig,