:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(all-destinations): Stop producer on exit (#127451)
Tomás Farías Santana9b443c9f58f4
diff --git a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.pyindex 6d4e385f93b6..dc323f728b43 100644--- a/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/azure_blob_batch_export.py@@ -3,7 +3,7 @@ import asyncioimport datetime as dtimport dataclassesfrom collections.abc import AsyncIterator-from contextlib import asynccontextmanager+from contextlib import AsyncExitStack, asynccontextmanagerfrom django.conf import settings@@ -346,7 +346,7 @@ async def insert_into_azure_blob_activity_from_stage(inputs: AzureBlobInsertInpublob_key,)- async with Heartbeater():+ async with Heartbeater(), AsyncExitStack() as cleanup:queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_AZURE_BLOB_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = ProducerFromInternalStage()@@ -359,6 +359,7 @@ async def insert_into_azure_blob_activity_from_stage(inputs: AzureBlobInsertInpumax_record_batch_size_bytes=1024 * 1024 * 60, # 60MBstage_folder=inputs.stage_folder,)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:diff --git a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.pyindex 50ecaf739ad9..ca25e186fb16 100644--- a/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/bigquery_batch_export.py@@ -1539,7 +1539,7 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)inputs.table_id,)- async with Heartbeater():+ async with Heartbeater(), contextlib.AsyncExitStack() as cleanup:model = migrate_batch_export_model(inputs.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_BIGQUERY_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)@@ -1553,6 +1553,7 @@ async def insert_into_bigquery_activity_from_stage(inputs: BigQueryInsertInputs)max_record_batch_size_bytes=1024 * 1024 * 60, # 60MBstage_folder=inputs.stage_folder,)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:diff --git a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.pyindex 0c6a221b4d2b..3b8767a9d465 100644--- a/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/databricks_batch_export.py@@ -1330,7 +1330,7 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInpinputs.table_name,)- async with Heartbeater():+ async with Heartbeater(), contextlib.AsyncExitStack() as cleanup:model = migrate_batch_export_model(inputs.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_DATABRICKS_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)@@ -1344,6 +1344,7 @@ async def insert_into_databricks_activity_from_stage(inputs: DatabricksInsertInpmax_record_batch_size_bytes=1024 * 1024 * 10, # 10MBstage_folder=inputs.stage_folder,)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:diff --git a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.pyindex ed059fc75ab3..4b827709fb04 100644--- a/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/postgres_batch_export.py@@ -986,7 +986,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)inputs.table_name,)- async with Heartbeater():+ async with Heartbeater(), contextlib.AsyncExitStack() as cleanup:model = migrate_batch_export_model(inputs.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_POSTGRES_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)@@ -1000,6 +1000,7 @@ async def insert_into_postgres_activity_from_stage(inputs: PostgresInsertInputs)max_record_batch_size_bytes=1024 * 1024 * 10, # 10MBstage_folder=inputs.stage_folder,)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:diff --git a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.pyindex a1aceee13e81..6209049b19d3 100644--- a/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/redshift_batch_export.py@@ -1266,7 +1266,7 @@ async def insert_into_redshift_activity_from_stage(inputs: RedshiftInsertInputs)inputs.table.name,)- async with Heartbeater():+ async with Heartbeater(), contextlib.AsyncExitStack() as cleanup:model = migrate_batch_export_model(inputs.batch_export.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_REDSHIFT_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)@@ -1280,6 +1280,7 @@ async def insert_into_redshift_activity_from_stage(inputs: RedshiftInsertInputs)max_record_batch_size_bytes=1024 * 1024 * 10, # 10MBstage_folder=inputs.batch_export.stage_folder,)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:@@ -1782,7 +1783,7 @@ async def copy_into_redshift_activity_from_stage(inputs: RedshiftCopyActivityInpinputs.table.name,)- async with Heartbeater():+ async with Heartbeater(), contextlib.AsyncExitStack() as cleanup:model = migrate_batch_export_model(inputs.batch_export.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)@@ -1796,6 +1797,7 @@ async def copy_into_redshift_activity_from_stage(inputs: RedshiftCopyActivityInpmax_record_batch_size_bytes=1024 * 1024 * 60, # 60MBstage_folder=inputs.batch_export.stage_folder,)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:diff --git a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py b/products/batch_exports/backend/temporal/destinations/s3_batch_export.pyindex 13e5cab92b14..2d6fe1289413 100644--- a/products/batch_exports/backend/temporal/destinations/s3_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/s3_batch_export.py@@ -757,83 +757,85 @@ async def insert_into_s3_from_stage(get_s3_key_from_inputs(inputs, file_name_prefix=file_name_prefix),)- queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)- producer = ProducerFromInternalStage()- assert inputs.batch_export_id is not None- producer_task = await producer.start(- queue=queue,- batch_export_id=inputs.batch_export_id,- data_interval_start=inputs.data_interval_start,- data_interval_end=inputs.data_interval_end,- max_record_batch_size_bytes=1024 * 1024 * 60, # 60MB- stage_folder=inputs.stage_folder,- )-- record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)- if record_batch_schema is None:- external_logger.info(- "Batch export will finish early as there is no data matching specified filters in range %s - %s",- inputs.data_interval_start or "START",- inputs.data_interval_end or "END",+ async with contextlib.AsyncExitStack() as cleanup:+ queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_S3_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)+ producer = ProducerFromInternalStage()+ assert inputs.batch_export_id is not None+ producer_task = await producer.start(+ queue=queue,+ batch_export_id=inputs.batch_export_id,+ data_interval_start=inputs.data_interval_start,+ data_interval_end=inputs.data_interval_end,+ max_record_batch_size_bytes=1024 * 1024 * 60, # 60MB+ stage_folder=inputs.stage_folder,)+ cleanup.push_async_callback(producer.stop)++ record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)+ if record_batch_schema is None:+ external_logger.info(+ "Batch export will finish early as there is no data matching specified filters in range %s - %s",+ inputs.data_interval_start or "START",+ inputs.data_interval_end or "END",+ )- return S3BatchExportResult(records_completed=0, bytes_exported=0)-- record_batch_schema = pa.schema(- # NOTE: For some reason, some batches set non-nullable fields as non-nullable, whereas other- # record batches have them as nullable.- # Until we figure it out, we set all fields to nullable. There are some fields we know- # are not nullable, but I'm opting for the more flexible option until we out why schemas differ- # between batches.- [field.with_nullable(True) for field in record_batch_schema]- )+ return S3BatchExportResult(records_completed=0, bytes_exported=0)- json_columns = ("properties", "person_properties", "set", "set_once")- if inputs.file_format.lower() == "jsonlines":- transformer = get_json_stream_transformer(- compression=inputs.compression,- include_inserted_at=True,- max_file_size_bytes=inputs.max_file_size_mb * 1024 * 1024 if inputs.max_file_size_mb else 0,- )- else:- transformer = ParquetStreamTransformer(- compression=inputs.compression,- include_inserted_at=True,- max_file_size_bytes=inputs.max_file_size_mb * 1024 * 1024 if inputs.max_file_size_mb else 0,+ record_batch_schema = pa.schema(+ # NOTE: For some reason, some batches set non-nullable fields as non-nullable, whereas other+ # record batches have them as nullable.+ # Until we figure it out, we set all fields to nullable. There are some fields we know+ # are not nullable, but I'm opting for the more flexible option until we out why schemas differ+ # between batches.+ [field.with_nullable(True) for field in record_batch_schema])- async with s3_client(- credentials,- use_virtual_style_addressing=inputs.use_virtual_style_addressing,- region=inputs.region,- endpoint_url=endpoint_url,- refresh_using=refresh_credentials,- ) as client:- consumer = ConcurrentS3Consumer.from_inputs(- s3_client=client,- s3_inputs=inputs,- part_size=settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES,- max_concurrent_uploads=settings.BATCH_EXPORT_S3_MAX_CONCURRENT_UPLOADS,- checksum_algorithm="CRC64NVME" if endpoint_url is None else None,- file_name_prefix=file_name_prefix,- )+ json_columns = ("properties", "person_properties", "set", "set_once")+ if inputs.file_format.lower() == "jsonlines":+ transformer = get_json_stream_transformer(+ compression=inputs.compression,+ include_inserted_at=True,+ max_file_size_bytes=inputs.max_file_size_mb * 1024 * 1024 if inputs.max_file_size_mb else 0,+ )+ else:+ transformer = ParquetStreamTransformer(+ compression=inputs.compression,+ include_inserted_at=True,+ max_file_size_bytes=inputs.max_file_size_mb * 1024 * 1024 if inputs.max_file_size_mb else 0,+ )- result = await run_consumer_from_stage(- queue=queue,- consumer=consumer,- producer_task=producer_task,- transformer=transformer,- json_columns=json_columns,- records_total=inputs.records_total,- )+ async with s3_client(+ credentials,+ use_virtual_style_addressing=inputs.use_virtual_style_addressing,+ region=inputs.region,+ endpoint_url=endpoint_url,+ refresh_using=refresh_credentials,+ ) as client:+ consumer = ConcurrentS3Consumer.from_inputs(+ s3_client=client,+ s3_inputs=inputs,+ part_size=settings.BATCH_EXPORT_S3_UPLOAD_CHUNK_SIZE_BYTES,+ max_concurrent_uploads=settings.BATCH_EXPORT_S3_MAX_CONCURRENT_UPLOADS,+ checksum_algorithm="CRC64NVME" if endpoint_url is None else None,+ file_name_prefix=file_name_prefix,+ )- return S3BatchExportResult(- bytes_exported=result.bytes_exported,- records_completed=result.records_completed,- records_failed=result.records_failed,- error=result.error,- files_uploaded=consumer.files_uploaded,- )+ result = await run_consumer_from_stage(+ queue=queue,+ consumer=consumer,+ producer_task=producer_task,+ transformer=transformer,+ json_columns=json_columns,+ records_total=inputs.records_total,+ )++ return S3BatchExportResult(+ bytes_exported=result.bytes_exported,+ records_completed=result.records_completed,+ records_failed=result.records_failed,+ error=result.error,+ files_uploaded=consumer.files_uploaded,+ )class ConcurrentS3Consumer(Consumer):diff --git a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.pyindex 17118d594052..9773e676622b 100644--- a/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/snowflake_batch_export.py@@ -1492,7 +1492,7 @@ async def insert_into_snowflake_activity_from_stage(inputs.table_name,)- async with Heartbeater():+ async with Heartbeater(), contextlib.AsyncExitStack() as cleanup:if inputs.integration_id is None:raise SnowflakeIntegrationRequiredError(inputs.batch_export_id)@@ -1511,6 +1511,7 @@ async def insert_into_snowflake_activity_from_stage(max_record_batch_size_bytes=1024 * 1024 * 10, # 10MBstage_folder=inputs.stage_folder,)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:diff --git a/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py b/products/batch_exports/backend/temporal/destinations/workflows_batch_export.pyindex 64504ee3e15d..f7c901e043e0 100644--- a/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py+++ b/products/batch_exports/backend/temporal/destinations/workflows_batch_export.py@@ -3,6 +3,7 @@ import timeimport typingimport asyncioimport datetime as dt+import contextlibimport dataclassesimport urllib.parse@@ -414,7 +415,7 @@ async def insert_into_workflows_activity_from_stage(inputs: WorkflowsInsertInputinputs.batch_export.data_interval_end or "END",)- async with Heartbeater():+ async with Heartbeater(), contextlib.AsyncExitStack() as cleanup:model = migrate_batch_export_model(inputs.batch_export.batch_export_model)queue = RecordBatchQueue(max_size_bytes=settings.BATCH_EXPORT_WORKFLOWS_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES)producer = Producer()@@ -426,6 +427,7 @@ async def insert_into_workflows_activity_from_stage(inputs: WorkflowsInsertInputdata_interval_end=inputs.batch_export.data_interval_end,max_record_batch_size_bytes=1024 * 1024 * 60, # 60MB)+ cleanup.push_async_callback(producer.stop)record_batch_schema = await wait_for_schema_or_producer(queue, producer_task)if record_batch_schema is None:diff --git a/products/batch_exports/backend/temporal/pipeline/producer.py b/products/batch_exports/backend/temporal/pipeline/producer.pyindex 093a931027cf..4e3c5ead5f24 100644--- a/products/batch_exports/backend/temporal/pipeline/producer.py+++ b/products/batch_exports/backend/temporal/pipeline/producer.py@@ -97,6 +97,14 @@ class Producer:)return self._task+ async def stop(self) -> None:+ if self._task is None:+ return+ if not self._task.done():+ self._task.cancel()+ # Cleanup must not replace the destination error that caused this producer to stop.+ await asyncio.gather(self._task, return_exceptions=True)+async def produce_batch_export_record_batches_from_range(self,queue: RecordBatchQueue,diff --git a/products/batch_exports/backend/tests/test_producer_cleanup.py b/products/batch_exports/backend/tests/test_producer_cleanup.pynew file mode 100644index 000000000000..48dbfd307f9d--- /dev/null+++ b/products/batch_exports/backend/tests/test_producer_cleanup.py@@ -0,0 +1,171 @@+import asyncio+import contextlib+from collections.abc import AsyncIterator, Iterator++import pytest+from unittest.mock import AsyncMock, MagicMock, patch++from django.test import override_settings++import pyarrow as pa+from temporalio.testing import ActivityEnvironment++from products.batch_exports.backend.temporal.destinations.bigquery_batch_export import (+ BigQueryClient,+ BigQueryInsertInputs,+ insert_into_bigquery_activity_from_stage,+)+from products.batch_exports.backend.temporal.pipeline.producer import Producer+from products.batch_exports.backend.temporal.queue import RecordBatchQueue++pytestmark = [pytest.mark.asyncio, pytest.mark.timeout(30)]+++class StagedExport:+ def __init__(self) -> None:+ batch = pa.record_batch({"amount": [1]})+ buffer = pa.BufferOutputStream()+ with pa.ipc.new_stream(buffer, batch.schema) as writer:+ writer.write_batch(batch)+ writer.write_batch(batch)+ self.data = buffer.getvalue().to_pybytes()+ self.reader_tasks: list[asyncio.Task[None]] = []+ self.producer_task: asyncio.Task[None] | None = None+ self.closing = asyncio.Event()+ self.closed = asyncio.Event()+ self.release_cleanup = asyncio.Event()+ self.release_cleanup.set()+ self.s3 = MagicMock()+ self.s3.list_objects_v2 = AsyncMock(return_value={"Contents": [{"Key": "stage/data.arrow"}]})+ body = MagicMock()+ body.iter_chunks.side_effect = self.iter_chunks+ self.s3.get_object = AsyncMock(+ return_value={"Body": body, "ContentLength": len(self.data), "ETag": '"test-etag"'}+ )++ async def iter_chunks(self, chunk_size: int) -> AsyncIterator[bytes]:+ task = asyncio.current_task()+ assert task is not None+ self.reader_tasks.append(task)+ yield self.data++ @contextlib.asynccontextmanager+ async def client(self) -> AsyncIterator[MagicMock]:+ self.producer_task = asyncio.current_task()+ try:+ yield self.s3+ finally:+ self.closing.set()+ await self.release_cleanup.wait()+ self.closed.set()+++def staged_export() -> Iterator[StagedExport]:+ staged = StagedExport()+ with patch("products.batch_exports.backend.temporal.pipeline.producer.get_s3_client", staged.client):+ yield staged+++async def test_stop_waits_for_blocked_readers_and_s3_cleanup(staged_export: StagedExport) -> None:+ producer = Producer()+ queue = RecordBatchQueue(max_size_bytes=1)+ await producer.start(queue, "test-export", None, None, stage_folder="stage/")+ try:+ await queue.get_schema()+ assert queue.full()+ assert not producer.task.done()+ staged_export.release_cleanup.clear()++ stop_task = asyncio.create_task(producer.stop())+ await staged_export.closing.wait()+ assert not stop_task.done()+ assert all(task.done() for task in staged_export.reader_tasks)++ staged_export.release_cleanup.set()+ await stop_task+ assert producer.task.cancelled()+ assert staged_export.closed.is_set()+ await producer.stop()+ finally:+ staged_export.release_cleanup.set()+ producer.task.cancel()+ await asyncio.gather(producer.task, return_exceptions=True)++[email protected]("failed", [False, True], ids=["completed", "failed"])+async def test_stop_preserves_finished_producer_result(staged_export: StagedExport, failed: bool) -> None:+ error = RuntimeError("Staging read failed")+ if failed:+ staged_export.s3.list_objects_v2.side_effect = error+ else:+ staged_export.s3.list_objects_v2.return_value = {}+ producer = Producer()+ await producer.start(RecordBatchQueue(), "test-export", None, None, stage_folder="stage/")+ await asyncio.wait([producer.task])++ await producer.stop()++ assert not producer.task.cancelled()+ assert producer.task.exception() is (error if failed else None)+ assert staged_export.closed.is_set()++[email protected]("cancelled", [False, True], ids=["setup-failure", "activity-cancellation"])+async def test_bigquery_activity_stops_producer_before_exiting(staged_export: StagedExport, cancelled: bool) -> None:+ entered = asyncio.Event()+ release = asyncio.Event()+ error = RuntimeError("Destination setup failed")++ async def get_or_create_table(table: object) -> None:+ entered.set()+ await release.wait()+ raise error++ client = MagicMock()+ client.__aenter__ = AsyncMock(return_value=client)+ client.__aexit__ = AsyncMock(return_value=False)+ client.get_or_create_table = get_or_create_table+ inputs = BigQueryInsertInputs(+ team_id=1,+ batch_export_id="test-export",+ data_interval_start="2026-01-01T00:00:00+00:00",+ data_interval_end="2026-01-01T01:00:00+00:00",+ stage_folder="stage/",+ project_id="test-project",+ dataset_id="test_dataset",+ table_id="test_table",+ private_key="unused-test-key",+ private_key_id="test-key-id",+ token_uri="https://example.com/token",+ client_email="[email protected]",+ )++ with (+ override_settings(BATCH_EXPORT_BIGQUERY_RECORD_BATCH_QUEUE_MAX_SIZE_BYTES=1),+ patch.object(BigQueryClient, "from_service_account_inputs", return_value=client),+ ):+ activity_task = asyncio.create_task(ActivityEnvironment().run(insert_into_bigquery_activity_from_stage, inputs))+ try:+ await entered.wait()+ assert staged_export.producer_task is not None+ assert not staged_export.producer_task.done()+ assert staged_export.reader_tasks+ if cancelled:+ activity_task.cancel()+ else:+ release.set()++ with pytest.raises(asyncio.CancelledError if cancelled else RuntimeError) as exc_info:+ await activity_task+ if not cancelled:+ assert exc_info.value is error+ assert staged_export.producer_task.cancelled()+ assert all(task.done() for task in staged_export.reader_tasks)+ assert staged_export.closed.is_set()+ finally:+ activity_task.cancel()+ await asyncio.gather(activity_task, return_exceptions=True)+ if staged_export.producer_task is not None:+ staged_export.producer_task.cancel()+ await asyncio.gather(staged_export.producer_task, return_exceptions=True)