:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(data-warehouse): throttle the warehouse cdp producer per table (#130275)
Tom Owers1d68080c1701
diff --git a/docs/published/handbook/engineering/data-warehouse.md b/docs/published/handbook/engineering/data-warehouse.mdindex d0dd25924220..e077ace2bb18 100644--- a/docs/published/handbook/engineering/data-warehouse.md+++ b/docs/published/handbook/engineering/data-warehouse.md@@ -10,6 +10,18 @@ Projects with a connected warehouse source or warehouse table use ordinary emptyOnboarding only checks warehouse setup for non-demo projects without events. Source detection returns an existence flag without serializing schemas, and table detection omits columns. Navigation refreshes an empty result so returning after source creation opens the Sources list.+## Warehouse CDP producer++Each team is limited to `ROWS_PER_SECOND` (2,000 rows per second) in `cdp_producer_rate.py`.+All producer runs of a team share one Redis bucket, across every table.++The producer deletes each staged file only after Kafka confirms delivery of its rows.+Checkpoints record progress after 10,000 produced rows or 30 seconds while processing rows.+Each file also has an end checkpoint.+Activity retries resume from the last heartbeat checkpoint.+Failed or canceled attempts retain unfinished files, so retries can produce duplicate rows.+The activity has a five-minute heartbeat timeout and permits up to ten attempts.+## Direct SQL validation**Send raw query** checks statement boundaries and read-only restrictions using the selected connection's SQL dialect. Quoted strings, identifiers, and comments follow that dialect, including backslashes in LIKE and regex patterns. Queries with unfinished quotes or comments are rejected before execution. For PostgreSQL, Redshift, and MySQL, quoting must have the same boundaries with and without session-level backslash escaping; use doubled quotes when embedding a quote in a string.diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.py b/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.pyindex 45ced6d5b5d8..3de03b815028 100644--- a/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.py+++ b/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.py@@ -7,10 +7,15 @@ from temporalio import activity, workflowfrom temporalio.common import RetryPolicyfrom posthog.temporal.common.base import PostHogWorkflow+from posthog.temporal.common.heartbeat import Heartbeaterfrom posthog.temporal.common.logger import get_loggerfrom posthog.temporal.utils import CDPProducerWorkflowInputs-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import CDPProducer+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import (+ CDPProducer,+ ProduceCheckpoint,+)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate import RowThrottleLOGGER = get_logger(__name__)@@ -38,7 +43,19 @@ async def produce_to_cdp_kafka_activity(inputs: CDPProducerWorkflowInputs) -> Noraise ValueError("CDP producer needs either a schema_id or a saved_query_id")try:- await producer.produce_to_kafka_from_s3()+ resume_from = ProduceCheckpoint.from_heartbeat_details(activity.info().heartbeat_details)+ throttle = RowThrottle(team_id=inputs.team_id)+ async with Heartbeater() as heartbeater:+ if resume_from is not None:+ heartbeater.details = resume_from.to_heartbeat_details()+ await logger.ainfo(+ "Resuming CDP production", file_path=resume_from.file_path, rows_done=resume_from.rows_done+ )+ await producer.produce_to_kafka_from_s3(+ resume_from=resume_from,+ on_checkpoint=lambda cp: setattr(heartbeater, "details", cp.to_heartbeat_details()),+ throttle=throttle,+ )except Exception:CDP_PRODUCER_RUNS_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()raise@@ -58,5 +75,11 @@ class CDPProducerJobWorkflow(PostHogWorkflow):produce_to_cdp_kafka_activity,inputs,start_to_close_timeout=timedelta(hours=24),- retry_policy=RetryPolicy(maximum_attempts=3),+ heartbeat_timeout=timedelta(minutes=5),+ retry_policy=RetryPolicy(+ maximum_attempts=10,+ initial_interval=timedelta(seconds=30),+ backoff_coefficient=2.0,+ maximum_interval=timedelta(minutes=10),+ ),)diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.pyindex cb8270b3e63f..da22e9613ec2 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.py@@ -19,6 +19,14 @@ later run, which is the behavior this store removes, and never a dropped one: ansuppressed because an earlier run produced it. Serializing the load through the commit would meanholding a lock across the whole produce loop, which runs for as long as the rows take to reachKafka, and a lock that expires under that would give the same interleaving with more to go wrong.++A run that fails part way is retried, either from its last checkpoint or, once the activity's+retries run out, by a fresh workflow attempt for the same job. Neither produces the rows of files an+earlier attempt already finished and deleted. So each checkpoint also adds the rows recorded so far+to an attempt key for the job, which every attempt of that job reads back and the commit clears.+Without it, the retry's commit would leave out the rows the earlier attempt delivered, and the next+run would trigger them again. The key is per job, so an overlapping run of the same view never+touches another run's progress."""import asyncio@@ -58,6 +66,12 @@ def emitted_rows_key(team_id: int, saved_query_id: str) -> str:return f"cdp_produced_view_rows:{{{team_id}:{saved_query_id}}}"+# Each checkpoint refreshes this, and checkpoints come at least every 30 seconds while a run produces,+# so it only has to cover the gap between attempts. The commit deletes the key, so the TTL only bounds+# what a run that never commits leaves behind.+ATTEMPT_KEY_TTL_SECONDS = 2 * 24 * 60 * 60++class EmittedRowStore:"""The event ids a view produced on its previous run.@@ -65,22 +79,35 @@ class EmittedRowStore:trigger. Failing closed would drop a real one."""- def __init__(self, key: str | None, logger: FilteringBoundLogger) -> None:+ def __init__(self, key: str | None, logger: FilteringBoundLogger, *, job_id: str = "") -> None:self._key = key+ self._attempt_key = f"{key}:attempt:{job_id}"self._logger = loggerself._previous: set[str] = set()self._current: set[str] = set()self._pending_deliveries: list[tuple[str, asyncio.Future[Any]]] = []+ self._unpersisted: list[str] = []self._enabled = key is not Noneself._at_limit = Falseasync def load(self) -> None:+ """Read the previous run's record.++ Also reads back the rows earlier attempts of this job recorded before their last checkpoint,+ since a retry does not produce those rows again. Every id there was delivered by this job or,+ for a suppressed repeat, by an earlier run.+ """if not self._enabled:return+ assert self._key is not Nonetry:- members = await get_async_client().smembers(self._key)- self._previous = {member.decode() if isinstance(member, bytes) else member for member in members}+ client = get_async_client()+ members = await client.smembers(self._key)+ self._previous = {_decode(member) for member in members}+ for member in await client.smembers(self._attempt_key):+ self._track(_decode(member))+ self._unpersisted.clear()except Exception as e:self._enabled = Falseawait self._logger.awarning(f"Could not read produced view rows; not suppressing repeats this run: {e}")@@ -145,11 +172,34 @@ class EmittedRowStore:self._track(event_id)def _track(self, event_id: str) -> None:+ if event_id in self._current:+ returnif len(self._current) < MAX_TRACKED_ROWS:self._current.add(event_id)+ self._unpersisted.append(event_id)else:self._at_limit = True+ async def persist_progress(self) -> None:+ """Add the rows recorded since the last call to the attempt key.++ Call it before a checkpoint, so a retry that resumes there still records the rows before it.+ A failure costs those rows a repeated trigger next run, like any other Redis problem here.+ """+ if not self._enabled or not self._unpersisted:+ return++ assert self._key is not None+ try:+ pipeline = get_async_client().pipeline(transaction=False)+ for chunk in batched(self._unpersisted, _WRITE_CHUNK_SIZE, strict=False):+ pipeline.sadd(self._attempt_key, *chunk)+ pipeline.expire(self._attempt_key, ATTEMPT_KEY_TTL_SECONDS)+ await pipeline.execute()+ except Exception as e:+ await self._logger.awarning(f"Could not checkpoint produced view rows; a retry may repeat them: {e}")+ self._unpersisted.clear()+async def commit(self) -> None:"""Replace the record with what this run saw.@@ -164,10 +214,11 @@ class EmittedRowStore:f"More than {MAX_TRACKED_ROWS} view rows in one run; rows past that may trigger again next run")+ assert self._key is not Nonetry:client = get_async_client()if not self._current:- await client.delete(self._key)+ await client.delete(self._key, self._attempt_key)returnscratch_key = f"{self._key}:writing"@@ -179,6 +230,11 @@ class EmittedRowStore:pipeline.sadd(scratch_key, *chunk)pipeline.expire(scratch_key, EMITTED_ROWS_TTL_SECONDS)pipeline.rename(scratch_key, self._key)+ pipeline.delete(self._attempt_key)await pipeline.execute()except Exception as e:await self._logger.awarning(f"Could not record produced view rows; repeats may not be suppressed: {e}")+++def _decode(member: bytes | str) -> str:+ return member.decode() if isinstance(member, bytes) else memberdiff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.pyindex 9f909f7d4db1..1ec5ec57309f 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py@@ -1,8 +1,10 @@import json+import timeimport uuidimport typingimport asyncioimport hashlib+from collections.abc import Callable, Sequencefrom django.conf import settingsfrom django.db.utils import OperationalError as DjangoOperationalError@@ -32,6 +34,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdpEmittedRowStore,emitted_rows_key,)+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate import RowThrottlefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store import (aretry_staged_delete,aretry_staged_read,@@ -42,8 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import PostHofrom products.warehouse_sources.backend.view_consumers import VIEW_TRIGGER_SOURCEfrom products.workflows.backend.facade.api import has_active_workflow_for_warehouse_table-# Per-file exceptions are swallowed (the file is deleted and the run continues), so a failed file-# is silently dropped rows. The outcome label is what makes that visible to alerting.+# File outcomes let alerts detect failures even when a later activity attempt succeeds.CDP_PRODUCER_FILES_TOTAL = Counter("warehouse_cdp_producer_files_total","Staged CDP files read from S3 and produced to Kafka, by outcome",@@ -79,6 +81,28 @@ class CDPTriggerTable:id: str+@frozen+class ProduceCheckpoint:+ file_path: str+ rows_done: int++ def to_heartbeat_details(self) -> tuple[str, int]:+ return self.file_path, self.rows_done++ @classmethod+ def from_heartbeat_details(cls, details: Sequence[object]) -> "ProduceCheckpoint | None":+ if not isinstance(details, Sequence) or len(details) != 2:+ return None+ file_path, rows_done = details+ if not isinstance(file_path, str) or type(rows_done) is not int or rows_done < 0:+ return None+ return cls(file_path=file_path, rows_done=rows_done)+++class CDPProducerDeliveryError(Exception):+ pass++class CDPProducer:team_id: inttable: CDPTriggerTable@@ -312,9 +336,15 @@ class CDPProducer:high-volume path as it was."""key = emitted_rows_key(self.team_id, self.table.id) if self.table.kind == "view" else None- return EmittedRowStore(key, self.logger)-- async def produce_to_kafka_from_s3(self) -> None:+ return EmittedRowStore(key, self.logger, job_id=self.job_id)++ async def produce_to_kafka_from_s3(+ self,+ *,+ resume_from: ProduceCheckpoint | None = None,+ on_checkpoint: Callable[[ProduceCheckpoint], None] | None = None,+ throttle: RowThrottle | None = None,+ ) -> None:fs = self._get_fs()await self.logger.adebug(f"Producing CDP data to Kafka from S3 path prefix {self._get_path_prefix()}")@@ -323,7 +353,7 @@ class CDPProducer:# destinations and workflows against their configured table without an extra lookup.dot_notated_table_name = await self.get_dot_notated_table_name()- files_to_produce = await self._list_files_to_produce()+ files_to_produce = sorted(await self._list_files_to_produce())await self.logger.adebug(f"Found {len(files_to_produce)} files to produce to Kafka")@@ -332,10 +362,30 @@ class CDPProducer:suppressed_rows = 0async with async_producer_scope(profile=KafkaClusterProfile.CYCLOTRON) as kafka_producer:++ async def checkpoint(+ file_path: str, row_offset: int, window_futures: list[asyncio.Future[object]]+ ) -> float:+ try:+ await kafka_producer.flush()+ await asyncio.gather(*window_futures)+ except Exception as e:+ raise CDPProducerDeliveryError(f"Kafka delivery failed for staged file {file_path}") from e+ window_futures.clear()+ emitted_rows.record_settled()+ await emitted_rows.persist_progress()+ if on_checkpoint is not None:+ on_checkpoint(ProduceCheckpoint(file_path=file_path, rows_done=row_offset))+ return time.monotonic()+for file_path in files_to_produce:await self.logger.adebug(f"Producing file {file_path} to Kafka")row_index = 0+ row_offset = 0+ skip = resume_from.rows_done if resume_from is not None and resume_from.file_path == file_path else 0+ window_futures: list[asyncio.Future[object]] = []+ last_checkpoint = time.monotonic()async def _open_staged_file(path: str = file_path) -> pa.NativeFile:return await asyncio.to_thread(fs.open_input_file, path)@@ -346,50 +396,59 @@ class CDPProducer:pf = pq.ParquetFile(f)for batch in pf.iter_batches(batch_size=10_000):+ if row_offset + batch.num_rows <= skip:+ row_offset += batch.num_rows+ continuefor row in batch.to_pylist():+ if row_offset < skip:+ row_offset += 1+ continueevent_id = self._build_event_id(row)if emitted_rows.is_repeat(event_id):suppressed_rows += 1- continue+ else:+ if throttle is not None:+ await throttle.acquire_row()+ row_as_props = {+ "team_id": self.team_id,+ "table_name": dot_notated_table_name,+ "table_type": self.table.kind,+ "event_id": event_id,+ "properties": row,+ }+ delivery = await kafka_producer.produce(+ topic=KAFKA_DWH_CDP_RAW_TABLE,+ data=row_as_props,+ value_serializer=self._serialize_json,+ )+ window_futures.append(delivery)+ emitted_rows.record_on_delivery(event_id, delivery)+ row_index += 1+ row_offset += 1+ if len(window_futures) >= 10_000 or time.monotonic() - last_checkpoint >= 30:+ last_checkpoint = await checkpoint(file_path, row_offset, window_futures)++ await checkpoint(file_path, row_offset, window_futures)+ await self.logger.adebug(f"Deleting file {file_path}")- row_as_props = {- "team_id": self.team_id,- "table_name": dot_notated_table_name,- "table_type": self.table.kind,- "event_id": event_id,- "properties": row,- }- delivery = await kafka_producer.produce(- topic=KAFKA_DWH_CDP_RAW_TABLE,- data=row_as_props,- value_serializer=self._serialize_json,- )- emitted_rows.record_on_delivery(event_id, delivery)- row_index += 1-- emitted_rows.record_settled()+ async def _delete_staged_file(path: str = file_path) -> None:+ await asyncio.to_thread(fs.delete_file, path)- await kafka_producer.flush()+ await aretry_staged_delete(_delete_staged_file, path=file_path, logger=self.logger)CDP_PRODUCER_FILES_TOTAL.labels(team_id=str(self.team_id), outcome="produced").inc()await self.logger.adebug(f"Finished producing file {file_path} to Kafka")except Exception as e:CDP_PRODUCER_FILES_TOTAL.labels(team_id=str(self.team_id), outcome="failed").inc()capture_exception(e)await self.logger.adebug(f"Error producing file {file_path} to Kafka: {e}")+ raisefinally:# A row is remembered only once Kafka confirms it, or the next run would# suppress a row that no subscriber received.emitted_rows.record_delivered()- # TODO(Gilbert09): have better row tracking so we can retry from a particular rowif row_index:CDP_PRODUCER_ROWS_TOTAL.labels(team_id=str(self.team_id)).inc(row_index)await self.logger.adebug(f"Produced {row_index} rows")- await self.logger.adebug(f"Deleting file {file_path}")-- async def _delete_staged_file(path: str = file_path) -> None:- await asyncio.to_thread(fs.delete_file, path)-- await aretry_staged_delete(_delete_staged_file, path=file_path, logger=self.logger)await self.logger.adebug("Finished producing all CDP data to Kafka")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer_rate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer_rate.pynew file mode 100644index 000000000000..1cf7a83e88a1--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer_rate.py@@ -0,0 +1,50 @@+import asyncio+from collections.abc import Awaitable, Callable++from prometheus_client import Counter++from posthog.egress.limiter.outbound import OutboundRateLimiter, get_outbound_rate_limiter+from posthog.egress.limiter.policies import RatePolicy, register_policy++_RATE_DOMAIN = "warehouse_cdp_producer"+ROWS_PER_SECOND = 2000.0++CDP_PRODUCER_THROTTLE_WAITS_TOTAL = Counter(+ "warehouse_cdp_producer_throttle_waits_total",+ "Warehouse CDP producer waits for the table rate limit",+ labelnames=["team_id"],+)+++def rate_limits(rows_per_second: float) -> tuple[int, float]:+ if rows_per_second >= 1:+ return int(rows_per_second), 1.0+ return 1, 1.0 / rows_per_second+++register_policy(_RATE_DOMAIN, RatePolicy(limits=(rate_limits(ROWS_PER_SECOND),), in_memory_divider=1, reserve={}))+++class RowThrottle:+ def __init__(+ self,+ *,+ team_id: int,+ limiter: OutboundRateLimiter | None = None,+ sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,+ ) -> None:+ self._team_id = team_id+ self._key = f"{_RATE_DOMAIN}:team:{team_id}"+ self._limiter = limiter if limiter is not None else get_outbound_rate_limiter()+ self._sleep = sleep+ self._count, self._period = rate_limits(ROWS_PER_SECOND)+ self._block = max(1, min(self._count // 10, 500))+ self._allowance = 0++ async def acquire_row(self) -> None:+ if self._allowance == 0:+ while not await self._limiter.acquire(self._key, self._block, source="warehouse_cdp_producer"):+ CDP_PRODUCER_THROTTLE_WAITS_TOTAL.labels(team_id=str(self._team_id)).inc()+ await self._sleep(max(self._block * self._period / self._count, 0.05))+ self._allowance = self._block+ self._allowance -= 1diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.pyindex 971636ac9f93..07805f1f3794 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py@@ -1,6 +1,7 @@import jsonimport uuidimport asyncio+from collections.abc import Callablefrom contextlib import asynccontextmanagerfrom decimal import Decimalfrom io import BytesIO@@ -29,7 +30,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdpEmittedRowStore,emitted_rows_key,)-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import CDPProducer+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import (+ CDPProducer,+ CDPProducerDeliveryError,+ ProduceCheckpoint,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store import (ObjectStoreConfigurationError,)@@ -52,6 +57,15 @@ def _patch_async_producer_scope(mock_producer):)+def _successful_produce_mock() -> mock.AsyncMock:+ def delivery(**_kwargs: object) -> asyncio.Future[None]:+ future: asyncio.Future[None] = asyncio.get_running_loop().create_future()+ future.set_result(None)+ return future++ return mock.AsyncMock(side_effect=delivery)++@pytest.mark.django_db(transaction=True)@pytest.mark.asyncioasync def test_should_run_no_hog_function(team):@@ -381,7 +395,7 @@ async def test_produce_to_kafka_from_s3_includes_table_name(mock_get_s3_client,mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -436,7 +450,7 @@ async def test_produce_to_kafka_from_s3_success(mock_get_s3_client, team):mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -499,7 +513,7 @@ async def test_produce_to_kafka_from_s3_with_no_files(mock_get_s3_client, team):mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -521,7 +535,8 @@ async def test_produce_to_kafka_from_s3_with_no_files(mock_get_s3_client, team):@pytest.mark.asyncio@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.capture_exception")-async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, mock_get_s3_client, team):[email protected]("failure", ["produce", "delivery", "flush", "cancelled"])+async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, mock_get_s3_client, team, failure):source = await sync_to_async(ExternalDataSource.objects.create)(team=team, source_type=ExternalDataSourceType.POSTGRES)@@ -538,9 +553,24 @@ async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, momock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock(side_effect=Exception("Kafka connection failed"))+ delivery: asyncio.Future[None] = asyncio.get_running_loop().create_future()+ if failure == "delivery":+ delivery.set_exception(RuntimeError("Kafka delivery failed"))+ else:+ delivery.set_result(None)+ mock_kafka_producer.produce = mock.AsyncMock(return_value=delivery)mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()+ expected_error: type[BaseException] = CDPProducerDeliveryError+ if failure == "produce":+ mock_kafka_producer.produce.side_effect = RuntimeError("Kafka connection failed")+ expected_error = RuntimeError+ elif failure == "flush":+ mock_kafka_producer.flush.side_effect = RuntimeError("Kafka flush failed")+ elif failure == "cancelled":+ mock_kafka_producer.flush.side_effect = asyncio.CancelledError()+ expected_error = asyncio.CancelledError+ on_checkpoint = mock.Mock()test_data = pa.table({"id": [1], "name": ["Alice"]})parquet_buffer = BytesIO()@@ -561,11 +591,19 @@ async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, mowith (patch.object(producer, "_get_fs", return_value=mock_fs),_patch_async_producer_scope(mock_kafka_producer),+ pytest.raises(expected_error) as exc,):- await producer.produce_to_kafka_from_s3()+ await producer.produce_to_kafka_from_s3(on_checkpoint=on_checkpoint)- mock_capture_exception.assert_called_once()- mock_fs.delete_file.assert_called_once()+ if failure == "cancelled":+ mock_capture_exception.assert_not_called()+ else:+ mock_capture_exception.assert_called_once_with(exc.value)+ if failure in ("delivery", "flush"):+ assert "path/chunk_0.parquet" in str(exc.value)+ assert isinstance(exc.value.__cause__, RuntimeError)+ mock_fs.delete_file.assert_not_called()+ on_checkpoint.assert_not_called()@pytest.mark.django_db(transaction=True)@@ -589,7 +627,7 @@ async def test_produce_to_kafka_from_s3_s3_read_failure(mock_capture_exception,mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -604,12 +642,13 @@ async def test_produce_to_kafka_from_s3_s3_read_failure(mock_capture_exception,with (patch.object(producer, "_get_fs", return_value=mock_fs),_patch_async_producer_scope(mock_kafka_producer),+ pytest.raises(Exception, match="S3 read failed"),):await producer.produce_to_kafka_from_s3()mock_capture_exception.assert_called_once()mock_kafka_producer.produce.assert_not_called()- mock_fs.delete_file.assert_called_once()+ mock_fs.delete_file.assert_not_called()@pytest.mark.django_db(transaction=True)@@ -639,7 +678,7 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_read_failure(mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -708,7 +747,7 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_delete_failure(mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -751,7 +790,8 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_delete_failure(@pytest.mark.django_db(transaction=True)@pytest.mark.asyncio@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")-async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, team):[email protected]("rows_done", [0, 10_002])+async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, team, rows_done):source = await sync_to_async(ExternalDataSource.objects.create)(team=team, source_type=ExternalDataSourceType.POSTGRES)@@ -768,7 +808,7 @@ async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, teamock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -787,18 +827,90 @@ async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, teaproducer = CDPProducer.for_source(team_id=team.id, schema_id=str(schema.id), job_id="test_job", logger=mock.AsyncMock())+ on_checkpoint = mock.Mock()with (patch.object(producer, "_get_fs", return_value=mock_fs),_patch_async_producer_scope(mock_kafka_producer),+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.time",+ MagicMock(monotonic=mock.Mock(return_value=0.0)),+ ),):- await producer.produce_to_kafka_from_s3()+ await producer.produce_to_kafka_from_s3(+ resume_from=ProduceCheckpoint(file_path="path/chunk_0.parquet", rows_done=rows_done),+ on_checkpoint=on_checkpoint,+ )- assert mock_kafka_producer.produce.call_count == 15000- mock_kafka_producer.flush.assert_called_once()+ assert [call.kwargs["data"]["properties"]["id"] for call in mock_kafka_producer.produce.call_args_list] == list(+ range(rows_done, 15_000)+ )+ expected_offsets = [10_000, 15_000] if rows_done == 0 else [15_000]+ assert mock_kafka_producer.flush.call_count == len(expected_offsets)+ assert on_checkpoint.call_args_list == [+ mock.call(ProduceCheckpoint(file_path="path/chunk_0.parquet", rows_done=offset)) for offset in expected_offsets+ ]mock_fs.delete_file.assert_called_once()[email protected]("resume_file", ["path/chunk_0.parquet", "path/missing.parquet"])[email protected]("seconds_per_row", [0.0, 31.0])+async def test_produce_to_kafka_from_s3_resumes(resume_file: str, seconds_per_row: float) -> None:+ producer = CDPProducer.for_source(team_id=1, schema_id="schema", job_id="job", logger=mock.AsyncMock())+ producer._table_name_cache = "postgres.table"+ files = ["path/chunk_0.parquet", "path/chunk_1.parquet"]+ buffers = []+ for ids in ([1, 2, 3], [4, 5, 6]):+ buffer = BytesIO()+ pq.write_table(pa.table({"id": ids}), buffer)+ buffer.seek(0)+ buffers.append(buffer)++ fs = MagicMock()+ fs.open_input_file.side_effect = buffers+ s3_client = MagicMock()+ s3_client._ls = mock.AsyncMock(return_value=[{"Key": path, "type": "file"} for path in reversed(files)])+ kafka = MagicMock(produce=_successful_produce_mock(), flush=mock.AsyncMock())+ clock = MagicMock(monotonic=mock.Mock(return_value=0.0))++ async def acquire_row() -> None:+ clock.monotonic.return_value += seconds_per_row++ throttle = MagicMock(acquire_row=mock.AsyncMock(side_effect=acquire_row))+ on_checkpoint = mock.Mock()+ with (+ patch.object(producer, "_get_fs", return_value=fs),+ patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client"+ ) as get_s3_client,+ patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.time", clock),+ _patch_async_producer_scope(kafka),+ ):+ get_s3_client.return_value.__aenter__ = mock.AsyncMock(return_value=s3_client)+ await producer.produce_to_kafka_from_s3(+ resume_from=ProduceCheckpoint(file_path=resume_file, rows_done=2),+ on_checkpoint=on_checkpoint,+ throttle=throttle,+ )++ skipped_rows = 2 if resume_file == files[0] else 0+ assert [call.kwargs["data"]["properties"]["id"] for call in kafka.produce.call_args_list] == list(+ range(skipped_rows + 1, 7)+ )+ assert throttle.acquire_row.await_count == 6 - skipped_rows+ expected_checkpoints: list[object] = []+ for file_index, file_path in enumerate(files):+ if seconds_per_row:+ first_offset = skipped_rows + 1 if file_index == 0 else 1+ expected_checkpoints.extend(+ mock.call(ProduceCheckpoint(file_path=file_path, rows_done=offset)) for offset in range(first_offset, 4)+ )+ expected_checkpoints.append(mock.call(ProduceCheckpoint(file_path=file_path, rows_done=3)))+ assert on_checkpoint.call_args_list == expected_checkpoints+ assert fs.delete_file.call_args_list == [mock.call(path) for path in files]++@pytest.mark.django_db(transaction=True)@pytest.mark.asyncioasync def test_stage_chunk(team):@@ -1366,7 +1478,7 @@ async def test_produce_to_kafka_from_s3_marks_view_rows(mock_get_s3_client, teammock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=None)mock_kafka_producer = MagicMock()- mock_kafka_producer.produce = mock.AsyncMock()+ mock_kafka_producer.produce = _successful_produce_mock()mock_kafka_producer.flush = mock.AsyncMock()mock_kafka_producer.close = mock.AsyncMock()@@ -1415,6 +1527,9 @@ async def _produce_staged_rows(produce_raises: bool = False,delivery_fails: bool = False,delivery_settles_on_flush: bool = False,+ delete_raises: bool = False,+ resume_from: ProduceCheckpoint | None = None,+ on_checkpoint: Callable[[ProduceCheckpoint], None] | None = None,) -> list[dict]:"""Stage `rows` as one chunk, run a whole produce cycle, and return the rows it produced.@@ -1458,8 +1573,9 @@ async def _produce_staged_rows(mock_file.__enter__ = MagicMock(return_value=parquet_buffer)mock_file.__exit__ = MagicMock(return_value=False)mock_fs.open_input_file.return_value = mock_file- mock_fs.delete_file = MagicMock()+ mock_fs.delete_file = MagicMock(side_effect=RuntimeError("delete failed") if delete_raises else None)+ throttle = MagicMock(acquire_row=mock.AsyncMock())with (patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client"@@ -1469,8 +1585,9 @@ async def _produce_staged_rows():mock_get_s3_client.return_value.__aenter__ = mock.AsyncMock(return_value=mock_s3_client)mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=None)- await producer.produce_to_kafka_from_s3()+ await producer.produce_to_kafka_from_s3(throttle=throttle, resume_from=resume_from, on_checkpoint=on_checkpoint)+ assert throttle.acquire_row.await_count == mock_kafka_producer.produce.call_countreturn [call[1]["data"]["properties"] for call in mock_kafka_producer.produce.call_args_list]@@ -1512,7 +1629,6 @@ async def test_what_a_later_view_run_produces(runs, produced_by_the_last_run):@pytest.mark.parametrize("failure",[- # A produce that raises is caught and the file dropped.{"produce_raises": True},# A produce only queues the row. Kafka reports a failed delivery later, on the future it returns.{"delivery_fails": True},@@ -1526,7 +1642,8 @@ async def test_a_view_row_whose_produce_failed_is_produced_on_the_next_run(failuview_id = str(uuid.uuid4())rows = [{"id": 1, "total": 5}]- await _produce_staged_rows(_view_producer_for_id(view_id), rows, **failure)+ with pytest.raises(Exception, match="Kafka"):+ await _produce_staged_rows(_view_producer_for_id(view_id), rows, **failure)assert await _produce_staged_rows(_view_producer_for_id(view_id, "job_2"), rows) == rows@@ -1543,6 +1660,24 @@ async def test_a_view_row_confirmed_only_at_flush_is_suppressed_on_the_next_run(assert await _produce_staged_rows(_view_producer_for_id(view_id, "job_2"), rows) == []+async def test_a_view_row_delivered_before_a_retried_checkpoint_is_suppressed_on_the_next_run():+ # A retry resumes after its checkpoint and never sees the rows before it. Its commit must still+ # record them, or the next run triggers them again.+ view_id = str(uuid.uuid4())+ rows = [{"id": 1, "total": 5}, {"id": 2, "total": 7}]+ checkpoints: list[ProduceCheckpoint] = []++ with pytest.raises(RuntimeError, match="delete failed"):+ await _produce_staged_rows(+ _view_producer_for_id(view_id), rows, delete_raises=True, on_checkpoint=checkpoints.append+ )++ assert checkpoints[-1].rows_done == len(rows)+ assert await _produce_staged_rows(_view_producer_for_id(view_id), rows, resume_from=checkpoints[-1]) == []+ assert await _produce_staged_rows(_view_producer_for_id(view_id, "job_2"), rows) == []++@pytest.mark.asyncioasync def test_a_source_row_is_produced_on_every_sync():# A source's event id already mixes in the job id, so the same row in a later sync is a new@@ -1592,6 +1727,30 @@ async def test_the_record_never_holds_more_than_the_tracked_row_limit():assert await get_async_client().scard(key) == 2+async def test_a_fresh_attempt_of_the_same_job_keeps_rows_an_earlier_attempt_checkpointed():+ # Once the activity's retries run out, the workflow retries the job with no heartbeat to resume+ # from. The files the earlier attempt finished are gone, so their rows must come from Redis.+ key = emitted_rows_key(1, str(uuid.uuid4()))+ earlier = EmittedRowStore(key, mock.AsyncMock(), job_id="job_1")+ await earlier.load()+ earlier.record_produced("a")+ await earlier.persist_progress()++ overlapping = EmittedRowStore(key, mock.AsyncMock(), job_id="job_2")+ await overlapping.load()++ retry = EmittedRowStore(key, mock.AsyncMock(), job_id="job_1")+ await retry.load()+ retry.record_produced("b")+ await retry.commit()++ assert {+ member.decode() if isinstance(member, bytes) else member for member in await get_async_client().smembers(key)+ } == {"a", "b"}+ assert not await get_async_client().exists(f"{key}:attempt:job_1")++def test_the_emitted_rows_key_and_its_scratch_key_share_a_cluster_slot():key = emitted_rows_key(1, "01a0d0f3-b1c8-0000-1589-739b4a75582c")diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer_rate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer_rate.pynew file mode 100644index 000000000000..f3af6891a817--- /dev/null+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer_rate.py@@ -0,0 +1,66 @@+import math+from collections.abc import Sequence++import pytest+from unittest.mock import AsyncMock, Mock, call, patch++from posthog.egress.limiter.outbound import OutboundRateLimiter+from posthog.egress.limiter.policies import resolve_policy++from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import ProduceCheckpoint+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate import (+ ROWS_PER_SECOND,+ RowThrottle,+ rate_limits,+)++[email protected]("rate,expected", [(2000, (2000, 1.0)), (1, (1, 1.0)), (0.5, (1, 2.0))])+def test_rate_limits(rate: float, expected: tuple[int, float]) -> None:+ assert rate_limits(rate) == expected+++ "details,expected",+ [+ (("file", 3), ProduceCheckpoint(file_path="file", rows_done=3)),+ (["file", 0], ProduceCheckpoint(file_path="file", rows_done=0)),+ ((), None),+ (("file",), None),+ (("file", 3, "extra"), None),+ (("file", "3"), None),+ (("file", 3.0), None),+ (("file", True), None),+ (("file", -1), None),+ ((123, 3), None),+ ],+)+def test_checkpoint_from_heartbeat_details(details: Sequence[object], expected: ProduceCheckpoint | None) -> None:+ assert ProduceCheckpoint.from_heartbeat_details(details) == expected+++ "rate,rows,block,delay",+ [(2000, 401, 200, 0.1), (10_000, 1001, 500, 0.05), (1, 3, 1, 1.0), (0.5, 3, 1, 2.0)],+)+async def test_row_throttle_waits_and_reserves_blocks(rate: float, rows: int, block: int, delay: float) -> None:+ limiter = Mock(spec=OutboundRateLimiter)+ limiter.acquire = AsyncMock(side_effect=[False] + [True] * math.ceil(rows / block))+ sleep = AsyncMock()+ with patch(+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate.ROWS_PER_SECOND",+ rate,+ ):+ throttle = RowThrottle(team_id=1, limiter=limiter, sleep=sleep)+ for _ in range(rows):+ await throttle.acquire_row()++ sleep.assert_awaited_once_with(delay)+ assert limiter.acquire.await_args_list == [+ call("warehouse_cdp_producer:team:1", block, source="warehouse_cdp_producer")+ ] * (math.ceil(rows / block) + 1)+ policy = resolve_policy("warehouse_cdp_producer:team:1")+ assert policy.limits == (rate_limits(ROWS_PER_SECOND),)+ assert policy.reserve == {}+ assert policy.in_memory_divider == 1