Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(data-warehouse): throttle the warehouse cdp producer per table (#130275)

Tom Owers1d68080c1701
diff --git a/docs/published/handbook/engineering/data-warehouse.md b/docs/published/handbook/engineering/data-warehouse.md
index d0dd25924220..e077ace2bb18 100644
--- a/docs/published/handbook/engineering/data-warehouse.md
+++ b/docs/published/handbook/engineering/data-warehouse.md
@@ -10,6 +10,18 @@ Projects with a connected warehouse source or warehouse table use ordinary empty
Onboarding only checks warehouse setup for non-demo projects without events. Source detection returns an existence flag without serializing schemas, and table detection omits columns. Navigation refreshes an empty result so returning after source creation opens the Sources list.
+## Warehouse CDP producer
+
+Each team is limited to `ROWS_PER_SECOND` (2,000 rows per second) in `cdp_producer_rate.py`.
+All producer runs of a team share one Redis bucket, across every table.
+
+The producer deletes each staged file only after Kafka confirms delivery of its rows.
+Checkpoints record progress after 10,000 produced rows or 30 seconds while processing rows.
+Each file also has an end checkpoint.
+Activity retries resume from the last heartbeat checkpoint.
+Failed or canceled attempts retain unfinished files, so retries can produce duplicate rows.
+The activity has a five-minute heartbeat timeout and permits up to ten attempts.
+
## Direct SQL validation
**Send raw query** checks statement boundaries and read-only restrictions using the selected connection's SQL dialect. Quoted strings, identifiers, and comments follow that dialect, including backslashes in LIKE and regex patterns. Queries with unfinished quotes or comments are rejected before execution. For PostgreSQL, Redshift, and MySQL, quoting must have the same boundaries with and without session-level backslash escaping; use doubled quotes when embedding a quote in a string.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.py b/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.py
index 45ced6d5b5d8..3de03b815028 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/cdp_producer_job.py
@@ -7,10 +7,15 @@ from temporalio import activity, workflow
from temporalio.common import RetryPolicy
from posthog.temporal.common.base import PostHogWorkflow
+from posthog.temporal.common.heartbeat import Heartbeater
from posthog.temporal.common.logger import get_logger
from posthog.temporal.utils import CDPProducerWorkflowInputs
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import CDPProducer
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import (
+ CDPProducer,
+ ProduceCheckpoint,
+)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate import RowThrottle
LOGGER = get_logger(__name__)
@@ -38,7 +43,19 @@ async def produce_to_cdp_kafka_activity(inputs: CDPProducerWorkflowInputs) -> No
raise ValueError("CDP producer needs either a schema_id or a saved_query_id")
try:
- await producer.produce_to_kafka_from_s3()
+ resume_from = ProduceCheckpoint.from_heartbeat_details(activity.info().heartbeat_details)
+ throttle = RowThrottle(team_id=inputs.team_id)
+ async with Heartbeater() as heartbeater:
+ if resume_from is not None:
+ heartbeater.details = resume_from.to_heartbeat_details()
+ await logger.ainfo(
+ "Resuming CDP production", file_path=resume_from.file_path, rows_done=resume_from.rows_done
+ )
+ await producer.produce_to_kafka_from_s3(
+ resume_from=resume_from,
+ on_checkpoint=lambda cp: setattr(heartbeater, "details", cp.to_heartbeat_details()),
+ throttle=throttle,
+ )
except Exception:
CDP_PRODUCER_RUNS_TOTAL.labels(team_id=str(inputs.team_id), outcome="failed").inc()
raise
@@ -58,5 +75,11 @@ class CDPProducerJobWorkflow(PostHogWorkflow):
produce_to_cdp_kafka_activity,
inputs,
start_to_close_timeout=timedelta(hours=24),
- retry_policy=RetryPolicy(maximum_attempts=3),
+ heartbeat_timeout=timedelta(minutes=5),
+ retry_policy=RetryPolicy(
+ maximum_attempts=10,
+ initial_interval=timedelta(seconds=30),
+ backoff_coefficient=2.0,
+ maximum_interval=timedelta(minutes=10),
+ ),
)
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.py
index cb8270b3e63f..da22e9613ec2 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_emitted_rows.py
@@ -19,6 +19,14 @@ later run, which is the behavior this store removes, and never a dropped one: an
suppressed because an earlier run produced it. Serializing the load through the commit would mean
holding a lock across the whole produce loop, which runs for as long as the rows take to reach
Kafka, and a lock that expires under that would give the same interleaving with more to go wrong.
+
+A run that fails part way is retried, either from its last checkpoint or, once the activity's
+retries run out, by a fresh workflow attempt for the same job. Neither produces the rows of files an
+earlier attempt already finished and deleted. So each checkpoint also adds the rows recorded so far
+to an attempt key for the job, which every attempt of that job reads back and the commit clears.
+Without it, the retry's commit would leave out the rows the earlier attempt delivered, and the next
+run would trigger them again. The key is per job, so an overlapping run of the same view never
+touches another run's progress.
"""
import asyncio
@@ -58,6 +66,12 @@ def emitted_rows_key(team_id: int, saved_query_id: str) -> str:
return f"cdp_produced_view_rows:{{{team_id}:{saved_query_id}}}"
+# Each checkpoint refreshes this, and checkpoints come at least every 30 seconds while a run produces,
+# so it only has to cover the gap between attempts. The commit deletes the key, so the TTL only bounds
+# what a run that never commits leaves behind.
+ATTEMPT_KEY_TTL_SECONDS = 2 * 24 * 60 * 60
+
+
class EmittedRowStore:
"""The event ids a view produced on its previous run.
@@ -65,22 +79,35 @@ class EmittedRowStore:
trigger. Failing closed would drop a real one.
"""
- def __init__(self, key: str | None, logger: FilteringBoundLogger) -> None:
+ def __init__(self, key: str | None, logger: FilteringBoundLogger, *, job_id: str = "") -> None:
self._key = key
+ self._attempt_key = f"{key}:attempt:{job_id}"
self._logger = logger
self._previous: set[str] = set()
self._current: set[str] = set()
self._pending_deliveries: list[tuple[str, asyncio.Future[Any]]] = []
+ self._unpersisted: list[str] = []
self._enabled = key is not None
self._at_limit = False
async def load(self) -> None:
+ """Read the previous run's record.
+
+ Also reads back the rows earlier attempts of this job recorded before their last checkpoint,
+ since a retry does not produce those rows again. Every id there was delivered by this job or,
+ for a suppressed repeat, by an earlier run.
+ """
if not self._enabled:
return
+ assert self._key is not None
try:
- members = await get_async_client().smembers(self._key)
- self._previous = {member.decode() if isinstance(member, bytes) else member for member in members}
+ client = get_async_client()
+ members = await client.smembers(self._key)
+ self._previous = {_decode(member) for member in members}
+ for member in await client.smembers(self._attempt_key):
+ self._track(_decode(member))
+ self._unpersisted.clear()
except Exception as e:
self._enabled = False
await self._logger.awarning(f"Could not read produced view rows; not suppressing repeats this run: {e}")
@@ -145,11 +172,34 @@ class EmittedRowStore:
self._track(event_id)
def _track(self, event_id: str) -> None:
+ if event_id in self._current:
+ return
if len(self._current) < MAX_TRACKED_ROWS:
self._current.add(event_id)
+ self._unpersisted.append(event_id)
else:
self._at_limit = True
+ async def persist_progress(self) -> None:
+ """Add the rows recorded since the last call to the attempt key.
+
+ Call it before a checkpoint, so a retry that resumes there still records the rows before it.
+ A failure costs those rows a repeated trigger next run, like any other Redis problem here.
+ """
+ if not self._enabled or not self._unpersisted:
+ return
+
+ assert self._key is not None
+ try:
+ pipeline = get_async_client().pipeline(transaction=False)
+ for chunk in batched(self._unpersisted, _WRITE_CHUNK_SIZE, strict=False):
+ pipeline.sadd(self._attempt_key, *chunk)
+ pipeline.expire(self._attempt_key, ATTEMPT_KEY_TTL_SECONDS)
+ await pipeline.execute()
+ except Exception as e:
+ await self._logger.awarning(f"Could not checkpoint produced view rows; a retry may repeat them: {e}")
+ self._unpersisted.clear()
+
async def commit(self) -> None:
"""Replace the record with what this run saw.
@@ -164,10 +214,11 @@ class EmittedRowStore:
f"More than {MAX_TRACKED_ROWS} view rows in one run; rows past that may trigger again next run"
)
+ assert self._key is not None
try:
client = get_async_client()
if not self._current:
- await client.delete(self._key)
+ await client.delete(self._key, self._attempt_key)
return
scratch_key = f"{self._key}:writing"
@@ -179,6 +230,11 @@ class EmittedRowStore:
pipeline.sadd(scratch_key, *chunk)
pipeline.expire(scratch_key, EMITTED_ROWS_TTL_SECONDS)
pipeline.rename(scratch_key, self._key)
+ pipeline.delete(self._attempt_key)
await pipeline.execute()
except Exception as e:
await self._logger.awarning(f"Could not record produced view rows; repeats may not be suppressed: {e}")
+
+
+def _decode(member: bytes | str) -> str:
+ return member.decode() if isinstance(member, bytes) else member
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
index 9f909f7d4db1..1ec5ec57309f 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer.py
@@ -1,8 +1,10 @@
import json
+import time
import uuid
import typing
import asyncio
import hashlib
+from collections.abc import Callable, Sequence
from django.conf import settings
from django.db.utils import OperationalError as DjangoOperationalError
@@ -32,6 +34,7 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp
EmittedRowStore,
emitted_rows_key,
)
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate import RowThrottle
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store import (
aretry_staged_delete,
aretry_staged_read,
@@ -42,8 +45,7 @@ from products.warehouse_sources.backend.temporal.data_imports.util import PostHo
from products.warehouse_sources.backend.view_consumers import VIEW_TRIGGER_SOURCE
from products.workflows.backend.facade.api import has_active_workflow_for_warehouse_table
-# Per-file exceptions are swallowed (the file is deleted and the run continues), so a failed file
-# is silently dropped rows. The outcome label is what makes that visible to alerting.
+# File outcomes let alerts detect failures even when a later activity attempt succeeds.
CDP_PRODUCER_FILES_TOTAL = Counter(
"warehouse_cdp_producer_files_total",
"Staged CDP files read from S3 and produced to Kafka, by outcome",
@@ -79,6 +81,28 @@ class CDPTriggerTable:
id: str
+@frozen
+class ProduceCheckpoint:
+ file_path: str
+ rows_done: int
+
+ def to_heartbeat_details(self) -> tuple[str, int]:
+ return self.file_path, self.rows_done
+
+ @classmethod
+ def from_heartbeat_details(cls, details: Sequence[object]) -> "ProduceCheckpoint | None":
+ if not isinstance(details, Sequence) or len(details) != 2:
+ return None
+ file_path, rows_done = details
+ if not isinstance(file_path, str) or type(rows_done) is not int or rows_done < 0:
+ return None
+ return cls(file_path=file_path, rows_done=rows_done)
+
+
+class CDPProducerDeliveryError(Exception):
+ pass
+
+
class CDPProducer:
team_id: int
table: CDPTriggerTable
@@ -312,9 +336,15 @@ class CDPProducer:
high-volume path as it was.
"""
key = emitted_rows_key(self.team_id, self.table.id) if self.table.kind == "view" else None
- return EmittedRowStore(key, self.logger)
-
- async def produce_to_kafka_from_s3(self) -> None:
+ return EmittedRowStore(key, self.logger, job_id=self.job_id)
+
+ async def produce_to_kafka_from_s3(
+ self,
+ *,
+ resume_from: ProduceCheckpoint | None = None,
+ on_checkpoint: Callable[[ProduceCheckpoint], None] | None = None,
+ throttle: RowThrottle | None = None,
+ ) -> None:
fs = self._get_fs()
await self.logger.adebug(f"Producing CDP data to Kafka from S3 path prefix {self._get_path_prefix()}")
@@ -323,7 +353,7 @@ class CDPProducer:
# destinations and workflows against their configured table without an extra lookup.
dot_notated_table_name = await self.get_dot_notated_table_name()
- files_to_produce = await self._list_files_to_produce()
+ files_to_produce = sorted(await self._list_files_to_produce())
await self.logger.adebug(f"Found {len(files_to_produce)} files to produce to Kafka")
@@ -332,10 +362,30 @@ class CDPProducer:
suppressed_rows = 0
async with async_producer_scope(profile=KafkaClusterProfile.CYCLOTRON) as kafka_producer:
+
+ async def checkpoint(
+ file_path: str, row_offset: int, window_futures: list[asyncio.Future[object]]
+ ) -> float:
+ try:
+ await kafka_producer.flush()
+ await asyncio.gather(*window_futures)
+ except Exception as e:
+ raise CDPProducerDeliveryError(f"Kafka delivery failed for staged file {file_path}") from e
+ window_futures.clear()
+ emitted_rows.record_settled()
+ await emitted_rows.persist_progress()
+ if on_checkpoint is not None:
+ on_checkpoint(ProduceCheckpoint(file_path=file_path, rows_done=row_offset))
+ return time.monotonic()
+
for file_path in files_to_produce:
await self.logger.adebug(f"Producing file {file_path} to Kafka")
row_index = 0
+ row_offset = 0
+ skip = resume_from.rows_done if resume_from is not None and resume_from.file_path == file_path else 0
+ window_futures: list[asyncio.Future[object]] = []
+ last_checkpoint = time.monotonic()
async def _open_staged_file(path: str = file_path) -> pa.NativeFile:
return await asyncio.to_thread(fs.open_input_file, path)
@@ -346,50 +396,59 @@ class CDPProducer:
pf = pq.ParquetFile(f)
for batch in pf.iter_batches(batch_size=10_000):
+ if row_offset + batch.num_rows <= skip:
+ row_offset += batch.num_rows
+ continue
for row in batch.to_pylist():
+ if row_offset < skip:
+ row_offset += 1
+ continue
event_id = self._build_event_id(row)
if emitted_rows.is_repeat(event_id):
suppressed_rows += 1
- continue
+ else:
+ if throttle is not None:
+ await throttle.acquire_row()
+ row_as_props = {
+ "team_id": self.team_id,
+ "table_name": dot_notated_table_name,
+ "table_type": self.table.kind,
+ "event_id": event_id,
+ "properties": row,
+ }
+ delivery = await kafka_producer.produce(
+ topic=KAFKA_DWH_CDP_RAW_TABLE,
+ data=row_as_props,
+ value_serializer=self._serialize_json,
+ )
+ window_futures.append(delivery)
+ emitted_rows.record_on_delivery(event_id, delivery)
+ row_index += 1
+ row_offset += 1
+ if len(window_futures) >= 10_000 or time.monotonic() - last_checkpoint >= 30:
+ last_checkpoint = await checkpoint(file_path, row_offset, window_futures)
+
+ await checkpoint(file_path, row_offset, window_futures)
+ await self.logger.adebug(f"Deleting file {file_path}")
- row_as_props = {
- "team_id": self.team_id,
- "table_name": dot_notated_table_name,
- "table_type": self.table.kind,
- "event_id": event_id,
- "properties": row,
- }
- delivery = await kafka_producer.produce(
- topic=KAFKA_DWH_CDP_RAW_TABLE,
- data=row_as_props,
- value_serializer=self._serialize_json,
- )
- emitted_rows.record_on_delivery(event_id, delivery)
- row_index += 1
-
- emitted_rows.record_settled()
+ async def _delete_staged_file(path: str = file_path) -> None:
+ await asyncio.to_thread(fs.delete_file, path)
- await kafka_producer.flush()
+ await aretry_staged_delete(_delete_staged_file, path=file_path, logger=self.logger)
CDP_PRODUCER_FILES_TOTAL.labels(team_id=str(self.team_id), outcome="produced").inc()
await self.logger.adebug(f"Finished producing file {file_path} to Kafka")
except Exception as e:
CDP_PRODUCER_FILES_TOTAL.labels(team_id=str(self.team_id), outcome="failed").inc()
capture_exception(e)
await self.logger.adebug(f"Error producing file {file_path} to Kafka: {e}")
+ raise
finally:
# A row is remembered only once Kafka confirms it, or the next run would
# suppress a row that no subscriber received.
emitted_rows.record_delivered()
- # TODO(Gilbert09): have better row tracking so we can retry from a particular row
if row_index:
CDP_PRODUCER_ROWS_TOTAL.labels(team_id=str(self.team_id)).inc(row_index)
await self.logger.adebug(f"Produced {row_index} rows")
- await self.logger.adebug(f"Deleting file {file_path}")
-
- async def _delete_staged_file(path: str = file_path) -> None:
- await asyncio.to_thread(fs.delete_file, path)
-
- await aretry_staged_delete(_delete_staged_file, path=file_path, logger=self.logger)
await self.logger.adebug("Finished producing all CDP data to Kafka")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer_rate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer_rate.py
new file mode 100644
index 000000000000..1cf7a83e88a1
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/cdp_producer_rate.py
@@ -0,0 +1,50 @@
+import asyncio
+from collections.abc import Awaitable, Callable
+
+from prometheus_client import Counter
+
+from posthog.egress.limiter.outbound import OutboundRateLimiter, get_outbound_rate_limiter
+from posthog.egress.limiter.policies import RatePolicy, register_policy
+
+_RATE_DOMAIN = "warehouse_cdp_producer"
+ROWS_PER_SECOND = 2000.0
+
+CDP_PRODUCER_THROTTLE_WAITS_TOTAL = Counter(
+ "warehouse_cdp_producer_throttle_waits_total",
+ "Warehouse CDP producer waits for the table rate limit",
+ labelnames=["team_id"],
+)
+
+
+def rate_limits(rows_per_second: float) -> tuple[int, float]:
+ if rows_per_second >= 1:
+ return int(rows_per_second), 1.0
+ return 1, 1.0 / rows_per_second
+
+
+register_policy(_RATE_DOMAIN, RatePolicy(limits=(rate_limits(ROWS_PER_SECOND),), in_memory_divider=1, reserve={}))
+
+
+class RowThrottle:
+ def __init__(
+ self,
+ *,
+ team_id: int,
+ limiter: OutboundRateLimiter | None = None,
+ sleep: Callable[[float], Awaitable[None]] = asyncio.sleep,
+ ) -> None:
+ self._team_id = team_id
+ self._key = f"{_RATE_DOMAIN}:team:{team_id}"
+ self._limiter = limiter if limiter is not None else get_outbound_rate_limiter()
+ self._sleep = sleep
+ self._count, self._period = rate_limits(ROWS_PER_SECOND)
+ self._block = max(1, min(self._count // 10, 500))
+ self._allowance = 0
+
+ async def acquire_row(self) -> None:
+ if self._allowance == 0:
+ while not await self._limiter.acquire(self._key, self._block, source="warehouse_cdp_producer"):
+ CDP_PRODUCER_THROTTLE_WAITS_TOTAL.labels(team_id=str(self._team_id)).inc()
+ await self._sleep(max(self._block * self._period / self._count, 0.05))
+ self._allowance = self._block
+ self._allowance -= 1
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
index 971636ac9f93..07805f1f3794 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer.py
@@ -1,6 +1,7 @@
import json
import uuid
import asyncio
+from collections.abc import Callable
from contextlib import asynccontextmanager
from decimal import Decimal
from io import BytesIO
@@ -29,7 +30,11 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp
EmittedRowStore,
emitted_rows_key,
)
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import CDPProducer
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import (
+ CDPProducer,
+ CDPProducerDeliveryError,
+ ProduceCheckpoint,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.staging_object_store import (
ObjectStoreConfigurationError,
)
@@ -52,6 +57,15 @@ def _patch_async_producer_scope(mock_producer):
)
+def _successful_produce_mock() -> mock.AsyncMock:
+ def delivery(**_kwargs: object) -> asyncio.Future[None]:
+ future: asyncio.Future[None] = asyncio.get_running_loop().create_future()
+ future.set_result(None)
+ return future
+
+ return mock.AsyncMock(side_effect=delivery)
+
+
@pytest.mark.django_db(transaction=True)
@pytest.mark.asyncio
async def test_should_run_no_hog_function(team):
@@ -381,7 +395,7 @@ async def test_produce_to_kafka_from_s3_includes_table_name(mock_get_s3_client,
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -436,7 +450,7 @@ async def test_produce_to_kafka_from_s3_success(mock_get_s3_client, team):
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -499,7 +513,7 @@ async def test_produce_to_kafka_from_s3_with_no_files(mock_get_s3_client, team):
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -521,7 +535,8 @@ async def test_produce_to_kafka_from_s3_with_no_files(mock_get_s3_client, team):
@pytest.mark.asyncio
@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")
@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.capture_exception")
-async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, mock_get_s3_client, team):
[email protected]("failure", ["produce", "delivery", "flush", "cancelled"])
+async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, mock_get_s3_client, team, failure):
source = await sync_to_async(ExternalDataSource.objects.create)(
team=team, source_type=ExternalDataSourceType.POSTGRES
)
@@ -538,9 +553,24 @@ async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, mo
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock(side_effect=Exception("Kafka connection failed"))
+ delivery: asyncio.Future[None] = asyncio.get_running_loop().create_future()
+ if failure == "delivery":
+ delivery.set_exception(RuntimeError("Kafka delivery failed"))
+ else:
+ delivery.set_result(None)
+ mock_kafka_producer.produce = mock.AsyncMock(return_value=delivery)
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
+ expected_error: type[BaseException] = CDPProducerDeliveryError
+ if failure == "produce":
+ mock_kafka_producer.produce.side_effect = RuntimeError("Kafka connection failed")
+ expected_error = RuntimeError
+ elif failure == "flush":
+ mock_kafka_producer.flush.side_effect = RuntimeError("Kafka flush failed")
+ elif failure == "cancelled":
+ mock_kafka_producer.flush.side_effect = asyncio.CancelledError()
+ expected_error = asyncio.CancelledError
+ on_checkpoint = mock.Mock()
test_data = pa.table({"id": [1], "name": ["Alice"]})
parquet_buffer = BytesIO()
@@ -561,11 +591,19 @@ async def test_produce_to_kafka_from_s3_kafka_failure(mock_capture_exception, mo
with (
patch.object(producer, "_get_fs", return_value=mock_fs),
_patch_async_producer_scope(mock_kafka_producer),
+ pytest.raises(expected_error) as exc,
):
- await producer.produce_to_kafka_from_s3()
+ await producer.produce_to_kafka_from_s3(on_checkpoint=on_checkpoint)
- mock_capture_exception.assert_called_once()
- mock_fs.delete_file.assert_called_once()
+ if failure == "cancelled":
+ mock_capture_exception.assert_not_called()
+ else:
+ mock_capture_exception.assert_called_once_with(exc.value)
+ if failure in ("delivery", "flush"):
+ assert "path/chunk_0.parquet" in str(exc.value)
+ assert isinstance(exc.value.__cause__, RuntimeError)
+ mock_fs.delete_file.assert_not_called()
+ on_checkpoint.assert_not_called()
@pytest.mark.django_db(transaction=True)
@@ -589,7 +627,7 @@ async def test_produce_to_kafka_from_s3_s3_read_failure(mock_capture_exception,
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -604,12 +642,13 @@ async def test_produce_to_kafka_from_s3_s3_read_failure(mock_capture_exception,
with (
patch.object(producer, "_get_fs", return_value=mock_fs),
_patch_async_producer_scope(mock_kafka_producer),
+ pytest.raises(Exception, match="S3 read failed"),
):
await producer.produce_to_kafka_from_s3()
mock_capture_exception.assert_called_once()
mock_kafka_producer.produce.assert_not_called()
- mock_fs.delete_file.assert_called_once()
+ mock_fs.delete_file.assert_not_called()
@pytest.mark.django_db(transaction=True)
@@ -639,7 +678,7 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_read_failure(
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -708,7 +747,7 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_delete_failure(
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -751,7 +790,8 @@ async def test_produce_to_kafka_from_s3_retries_a_transient_s3_delete_failure(
@pytest.mark.django_db(transaction=True)
@pytest.mark.asyncio
@patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client")
-async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, team):
[email protected]("rows_done", [0, 10_002])
+async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, team, rows_done):
source = await sync_to_async(ExternalDataSource.objects.create)(
team=team, source_type=ExternalDataSourceType.POSTGRES
)
@@ -768,7 +808,7 @@ async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, tea
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=False)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -787,18 +827,90 @@ async def test_produce_to_kafka_from_s3_with_large_batch(mock_get_s3_client, tea
producer = CDPProducer.for_source(
team_id=team.id, schema_id=str(schema.id), job_id="test_job", logger=mock.AsyncMock()
)
+ on_checkpoint = mock.Mock()
with (
patch.object(producer, "_get_fs", return_value=mock_fs),
_patch_async_producer_scope(mock_kafka_producer),
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.time",
+ MagicMock(monotonic=mock.Mock(return_value=0.0)),
+ ),
):
- await producer.produce_to_kafka_from_s3()
+ await producer.produce_to_kafka_from_s3(
+ resume_from=ProduceCheckpoint(file_path="path/chunk_0.parquet", rows_done=rows_done),
+ on_checkpoint=on_checkpoint,
+ )
- assert mock_kafka_producer.produce.call_count == 15000
- mock_kafka_producer.flush.assert_called_once()
+ assert [call.kwargs["data"]["properties"]["id"] for call in mock_kafka_producer.produce.call_args_list] == list(
+ range(rows_done, 15_000)
+ )
+ expected_offsets = [10_000, 15_000] if rows_done == 0 else [15_000]
+ assert mock_kafka_producer.flush.call_count == len(expected_offsets)
+ assert on_checkpoint.call_args_list == [
+ mock.call(ProduceCheckpoint(file_path="path/chunk_0.parquet", rows_done=offset)) for offset in expected_offsets
+ ]
mock_fs.delete_file.assert_called_once()
[email protected]("resume_file", ["path/chunk_0.parquet", "path/missing.parquet"])
[email protected]("seconds_per_row", [0.0, 31.0])
+async def test_produce_to_kafka_from_s3_resumes(resume_file: str, seconds_per_row: float) -> None:
+ producer = CDPProducer.for_source(team_id=1, schema_id="schema", job_id="job", logger=mock.AsyncMock())
+ producer._table_name_cache = "postgres.table"
+ files = ["path/chunk_0.parquet", "path/chunk_1.parquet"]
+ buffers = []
+ for ids in ([1, 2, 3], [4, 5, 6]):
+ buffer = BytesIO()
+ pq.write_table(pa.table({"id": ids}), buffer)
+ buffer.seek(0)
+ buffers.append(buffer)
+
+ fs = MagicMock()
+ fs.open_input_file.side_effect = buffers
+ s3_client = MagicMock()
+ s3_client._ls = mock.AsyncMock(return_value=[{"Key": path, "type": "file"} for path in reversed(files)])
+ kafka = MagicMock(produce=_successful_produce_mock(), flush=mock.AsyncMock())
+ clock = MagicMock(monotonic=mock.Mock(return_value=0.0))
+
+ async def acquire_row() -> None:
+ clock.monotonic.return_value += seconds_per_row
+
+ throttle = MagicMock(acquire_row=mock.AsyncMock(side_effect=acquire_row))
+ on_checkpoint = mock.Mock()
+ with (
+ patch.object(producer, "_get_fs", return_value=fs),
+ patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client"
+ ) as get_s3_client,
+ patch("products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.time", clock),
+ _patch_async_producer_scope(kafka),
+ ):
+ get_s3_client.return_value.__aenter__ = mock.AsyncMock(return_value=s3_client)
+ await producer.produce_to_kafka_from_s3(
+ resume_from=ProduceCheckpoint(file_path=resume_file, rows_done=2),
+ on_checkpoint=on_checkpoint,
+ throttle=throttle,
+ )
+
+ skipped_rows = 2 if resume_file == files[0] else 0
+ assert [call.kwargs["data"]["properties"]["id"] for call in kafka.produce.call_args_list] == list(
+ range(skipped_rows + 1, 7)
+ )
+ assert throttle.acquire_row.await_count == 6 - skipped_rows
+ expected_checkpoints: list[object] = []
+ for file_index, file_path in enumerate(files):
+ if seconds_per_row:
+ first_offset = skipped_rows + 1 if file_index == 0 else 1
+ expected_checkpoints.extend(
+ mock.call(ProduceCheckpoint(file_path=file_path, rows_done=offset)) for offset in range(first_offset, 4)
+ )
+ expected_checkpoints.append(mock.call(ProduceCheckpoint(file_path=file_path, rows_done=3)))
+ assert on_checkpoint.call_args_list == expected_checkpoints
+ assert fs.delete_file.call_args_list == [mock.call(path) for path in files]
+
+
@pytest.mark.django_db(transaction=True)
@pytest.mark.asyncio
async def test_stage_chunk(team):
@@ -1366,7 +1478,7 @@ async def test_produce_to_kafka_from_s3_marks_view_rows(mock_get_s3_client, team
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=None)
mock_kafka_producer = MagicMock()
- mock_kafka_producer.produce = mock.AsyncMock()
+ mock_kafka_producer.produce = _successful_produce_mock()
mock_kafka_producer.flush = mock.AsyncMock()
mock_kafka_producer.close = mock.AsyncMock()
@@ -1415,6 +1527,9 @@ async def _produce_staged_rows(
produce_raises: bool = False,
delivery_fails: bool = False,
delivery_settles_on_flush: bool = False,
+ delete_raises: bool = False,
+ resume_from: ProduceCheckpoint | None = None,
+ on_checkpoint: Callable[[ProduceCheckpoint], None] | None = None,
) -> list[dict]:
"""Stage `rows` as one chunk, run a whole produce cycle, and return the rows it produced.
@@ -1458,8 +1573,9 @@ async def _produce_staged_rows(
mock_file.__enter__ = MagicMock(return_value=parquet_buffer)
mock_file.__exit__ = MagicMock(return_value=False)
mock_fs.open_input_file.return_value = mock_file
- mock_fs.delete_file = MagicMock()
+ mock_fs.delete_file = MagicMock(side_effect=RuntimeError("delete failed") if delete_raises else None)
+ throttle = MagicMock(acquire_row=mock.AsyncMock())
with (
patch(
"products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer.aget_s3_client"
@@ -1469,8 +1585,9 @@ async def _produce_staged_rows(
):
mock_get_s3_client.return_value.__aenter__ = mock.AsyncMock(return_value=mock_s3_client)
mock_get_s3_client.return_value.__aexit__ = mock.AsyncMock(return_value=None)
- await producer.produce_to_kafka_from_s3()
+ await producer.produce_to_kafka_from_s3(throttle=throttle, resume_from=resume_from, on_checkpoint=on_checkpoint)
+ assert throttle.acquire_row.await_count == mock_kafka_producer.produce.call_count
return [call[1]["data"]["properties"] for call in mock_kafka_producer.produce.call_args_list]
@@ -1512,7 +1629,6 @@ async def test_what_a_later_view_run_produces(runs, produced_by_the_last_run):
@pytest.mark.parametrize(
"failure",
[
- # A produce that raises is caught and the file dropped.
{"produce_raises": True},
# A produce only queues the row. Kafka reports a failed delivery later, on the future it returns.
{"delivery_fails": True},
@@ -1526,7 +1642,8 @@ async def test_a_view_row_whose_produce_failed_is_produced_on_the_next_run(failu
view_id = str(uuid.uuid4())
rows = [{"id": 1, "total": 5}]
- await _produce_staged_rows(_view_producer_for_id(view_id), rows, **failure)
+ with pytest.raises(Exception, match="Kafka"):
+ await _produce_staged_rows(_view_producer_for_id(view_id), rows, **failure)
assert await _produce_staged_rows(_view_producer_for_id(view_id, "job_2"), rows) == rows
@@ -1543,6 +1660,24 @@ async def test_a_view_row_confirmed_only_at_flush_is_suppressed_on_the_next_run(
assert await _produce_staged_rows(_view_producer_for_id(view_id, "job_2"), rows) == []
+async def test_a_view_row_delivered_before_a_retried_checkpoint_is_suppressed_on_the_next_run():
+ # A retry resumes after its checkpoint and never sees the rows before it. Its commit must still
+ # record them, or the next run triggers them again.
+ view_id = str(uuid.uuid4())
+ rows = [{"id": 1, "total": 5}, {"id": 2, "total": 7}]
+ checkpoints: list[ProduceCheckpoint] = []
+
+ with pytest.raises(RuntimeError, match="delete failed"):
+ await _produce_staged_rows(
+ _view_producer_for_id(view_id), rows, delete_raises=True, on_checkpoint=checkpoints.append
+ )
+
+ assert checkpoints[-1].rows_done == len(rows)
+ assert await _produce_staged_rows(_view_producer_for_id(view_id), rows, resume_from=checkpoints[-1]) == []
+ assert await _produce_staged_rows(_view_producer_for_id(view_id, "job_2"), rows) == []
+
+
@pytest.mark.asyncio
async def test_a_source_row_is_produced_on_every_sync():
# A source's event id already mixes in the job id, so the same row in a later sync is a new
@@ -1592,6 +1727,30 @@ async def test_the_record_never_holds_more_than_the_tracked_row_limit():
assert await get_async_client().scard(key) == 2
+async def test_a_fresh_attempt_of_the_same_job_keeps_rows_an_earlier_attempt_checkpointed():
+ # Once the activity's retries run out, the workflow retries the job with no heartbeat to resume
+ # from. The files the earlier attempt finished are gone, so their rows must come from Redis.
+ key = emitted_rows_key(1, str(uuid.uuid4()))
+ earlier = EmittedRowStore(key, mock.AsyncMock(), job_id="job_1")
+ await earlier.load()
+ earlier.record_produced("a")
+ await earlier.persist_progress()
+
+ overlapping = EmittedRowStore(key, mock.AsyncMock(), job_id="job_2")
+ await overlapping.load()
+
+ retry = EmittedRowStore(key, mock.AsyncMock(), job_id="job_1")
+ await retry.load()
+ retry.record_produced("b")
+ await retry.commit()
+
+ assert {
+ member.decode() if isinstance(member, bytes) else member for member in await get_async_client().smembers(key)
+ } == {"a", "b"}
+ assert not await get_async_client().exists(f"{key}:attempt:job_1")
+
+
def test_the_emitted_rows_key_and_its_scratch_key_share_a_cluster_slot():
key = emitted_rows_key(1, "01a0d0f3-b1c8-0000-1589-739b4a75582c")
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer_rate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer_rate.py
new file mode 100644
index 000000000000..f3af6891a817
--- /dev/null
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_cdp_producer_rate.py
@@ -0,0 +1,66 @@
+import math
+from collections.abc import Sequence
+
+import pytest
+from unittest.mock import AsyncMock, Mock, call, patch
+
+from posthog.egress.limiter.outbound import OutboundRateLimiter
+from posthog.egress.limiter.policies import resolve_policy
+
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer import ProduceCheckpoint
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate import (
+ ROWS_PER_SECOND,
+ RowThrottle,
+ rate_limits,
+)
+
+
[email protected]("rate,expected", [(2000, (2000, 1.0)), (1, (1, 1.0)), (0.5, (1, 2.0))])
+def test_rate_limits(rate: float, expected: tuple[int, float]) -> None:
+ assert rate_limits(rate) == expected
+
+
+ "details,expected",
+ [
+ (("file", 3), ProduceCheckpoint(file_path="file", rows_done=3)),
+ (["file", 0], ProduceCheckpoint(file_path="file", rows_done=0)),
+ ((), None),
+ (("file",), None),
+ (("file", 3, "extra"), None),
+ (("file", "3"), None),
+ (("file", 3.0), None),
+ (("file", True), None),
+ (("file", -1), None),
+ ((123, 3), None),
+ ],
+)
+def test_checkpoint_from_heartbeat_details(details: Sequence[object], expected: ProduceCheckpoint | None) -> None:
+ assert ProduceCheckpoint.from_heartbeat_details(details) == expected
+
+
+ "rate,rows,block,delay",
+ [(2000, 401, 200, 0.1), (10_000, 1001, 500, 0.05), (1, 3, 1, 1.0), (0.5, 3, 1, 2.0)],
+)
+async def test_row_throttle_waits_and_reserves_blocks(rate: float, rows: int, block: int, delay: float) -> None:
+ limiter = Mock(spec=OutboundRateLimiter)
+ limiter.acquire = AsyncMock(side_effect=[False] + [True] * math.ceil(rows / block))
+ sleep = AsyncMock()
+ with patch(
+ "products.warehouse_sources.backend.temporal.data_imports.pipelines.core.cdp_producer_rate.ROWS_PER_SECOND",
+ rate,
+ ):
+ throttle = RowThrottle(team_id=1, limiter=limiter, sleep=sleep)
+ for _ in range(rows):
+ await throttle.acquire_row()
+
+ sleep.assert_awaited_once_with(delay)
+ assert limiter.acquire.await_args_list == [
+ call("warehouse_cdp_producer:team:1", block, source="warehouse_cdp_producer")
+ ] * (math.ceil(rows / block) + 1)
+ policy = resolve_policy("warehouse_cdp_producer:team:1")
+ assert policy.limits == (rate_limits(ROWS_PER_SECOND),)
+ assert policy.reserve == {}
+ assert policy.in_memory_divider == 1