:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(data-warehouse): build source items off the import event loop (#127584)
Tom Owersfe196f648c11
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.pyindex d3eba8931977..939a35340c42 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.py@@ -1,8 +1,8 @@import asyncioimport threadingimport contextvars-from collections.abc import AsyncIterable, AsyncIterator, Iterable-from concurrent.futures import ThreadPoolExecutor+from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable+from concurrent.futures import Future, ThreadPoolExecutorfrom typing import TypeVar, castT = TypeVar("T")@@ -13,6 +13,38 @@ T = TypeVar("T")_SOURCE_ITERATOR_EXECUTOR = ThreadPoolExecutor(max_workers=32, thread_name_prefix="source-iter")+def _close_abandoned_result(future: Future) -> None:+ if future.cancelled() or future.exception() is not None:+ return+ close = getattr(future.result(), "close", None)+ if callable(close):+ try:+ close()+ except Exception:+ pass+++async def run_in_source_executor(func: Callable[[], T]) -> T:+ """+ Run a blocking source call on the dedicated source thread pool, keeping contextvars.++ A running thread can't be interrupted, so if the caller is cancelled the call finishes+ in the background. Its result is then closed when it has a `close()`, so an abandoned+ generator doesn't hold cursors or connections until garbage collection.+ """+ ctx = contextvars.copy_context()++ def call() -> T:+ return ctx.run(func)++ future = _SOURCE_ITERATOR_EXECUTOR.submit(call)+ try:+ return await asyncio.wrap_future(future)+ except asyncio.CancelledError:+ future.add_done_callback(_close_abandoned_result)+ raise++async def async_iterate(iterable: Iterable[T] | AsyncIterable[T]) -> AsyncIterator[T]:"""Normalize a sync or async iterable into an async iterator.diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.pyindex bd764dc8e22b..3bddc7fcf6eb 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.py@@ -5,7 +5,10 @@ import contextvarsimport pytest-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import async_iterate+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import (+ async_iterate,+ run_in_source_executor,+)_probe: contextvars.ContextVar[str | None] = contextvars.ContextVar("probe", default=None)@@ -92,3 +95,43 @@ async def test_async_iterate_close_runs_when_cancelled_mid_iteration():# close() must have run despite cancellation mid-iteration.assert closed_flag.wait(timeout=3.0), "iterator.close() was not called"+++async def test_run_in_source_executor_closes_the_result_of_a_cancelled_call():+ started = threading.Event()+ release = threading.Event()+ closed = threading.Event()+ # Holding a reference stops garbage collection from closing the generator for us.+ built: list = []++ def build():+ started.set()+ release.wait(timeout=5)++ def gen():+ try:+ yield 1+ finally:+ closed.set()++ it = gen()+ next(it)+ built.append(it)+ return it++ task = asyncio.create_task(run_in_source_executor(build))+ await asyncio.to_thread(started.wait, 5)+ task.cancel()+ with pytest.raises(asyncio.CancelledError):+ await task++ release.set()+ assert await asyncio.to_thread(closed.wait, 5)+++async def test_run_in_source_executor_propagates_contextvars():+ _probe.set("bound-value")++ assert await run_in_source_executor(_probe.get) == "bound-value"diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.pyindex 453834bd82a1..107e55e286a4 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py@@ -86,7 +86,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arrreconcile_batch_to_accumulated_schema,source_uses_delta_write_column_selection,)-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import async_iterate+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import (+ async_iterate,+ run_in_source_executor,+)from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcherfrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenancefrom products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef@@ -998,7 +1001,10 @@ class PipelineV3(Generic[ResumableData]):):await self._record_table_rebuild(self._run_uuid)- items = self._resource.items()+ # A source can do blocking work when it builds its items, such as the DNS lookup of a+ # host check. The source thread pool keeps that work off the event loop, as iteration+ # already is, without tying up the default executor.+ items = await run_in_source_executor(self._resource.items)safe_point_scope = self._activate_safe_point(items)source_items = self._source_items(items, source_type)awaiting_source = Truediff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.pyindex bbc55b25c6ce..3cbda075a725 100644--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py@@ -3098,6 +3098,35 @@ def _staged_ids(pipeline: PipelineV3) -> list[str]:]+async def test_the_source_builds_its_items_off_the_event_loop() -> None:+ loop_thread = threading.get_ident()+ built_in: list[int] = []++ def items():+ built_in.append(threading.get_ident())+ return iter([pa.table({"id": ["a"]})])++ pipeline = _runnable_pipeline(_manager(), items)+ pipeline._finalize = AsyncMock() # type: ignore[method-assign]+ with ExitStack() as stack:+ for name in (+ "reset_rows_synced_if_needed",+ "setup_row_tracking_with_billing_check",+ "handle_reset_or_full_refresh",+ "handle_corrupted_delta_log",+ ):+ stack.enter_context(patch(f"{_PIPELINE}.{name}", new_callable=AsyncMock))+ for name in ("validate_incremental_sync", "record_source_item_stats"):+ stack.enter_context(patch(f"{_PIPELINE}.{name}"))+ stack.enter_context(patch.object(ResumableSourceManager, "_get_redis", lambda self: nullcontext(MagicMock())))+ stack.enter_context(patch(f"{_PIPELINE}.activity")).in_activity.return_value = False+ await pipeline.run()++ assert len(built_in) == 1+ assert built_in[0] != loop_thread++class TestSourcePreemption:@pytest.mark.parametrize("late_write", ["commit", "clear_state"])@pytest.mark.asyncio