Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(data-warehouse): build source items off the import event loop (#127584)

Tom Owersfe196f648c11
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.py
index d3eba8931977..939a35340c42 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/async_iterate.py
@@ -1,8 +1,8 @@
import asyncio
import threading
import contextvars
-from collections.abc import AsyncIterable, AsyncIterator, Iterable
-from concurrent.futures import ThreadPoolExecutor
+from collections.abc import AsyncIterable, AsyncIterator, Callable, Iterable
+from concurrent.futures import Future, ThreadPoolExecutor
from typing import TypeVar, cast
T = TypeVar("T")
@@ -13,6 +13,38 @@ T = TypeVar("T")
_SOURCE_ITERATOR_EXECUTOR = ThreadPoolExecutor(max_workers=32, thread_name_prefix="source-iter")
+def _close_abandoned_result(future: Future) -> None:
+ if future.cancelled() or future.exception() is not None:
+ return
+ close = getattr(future.result(), "close", None)
+ if callable(close):
+ try:
+ close()
+ except Exception:
+ pass
+
+
+async def run_in_source_executor(func: Callable[[], T]) -> T:
+ """
+ Run a blocking source call on the dedicated source thread pool, keeping contextvars.
+
+ A running thread can't be interrupted, so if the caller is cancelled the call finishes
+ in the background. Its result is then closed when it has a `close()`, so an abandoned
+ generator doesn't hold cursors or connections until garbage collection.
+ """
+ ctx = contextvars.copy_context()
+
+ def call() -> T:
+ return ctx.run(func)
+
+ future = _SOURCE_ITERATOR_EXECUTOR.submit(call)
+ try:
+ return await asyncio.wrap_future(future)
+ except asyncio.CancelledError:
+ future.add_done_callback(_close_abandoned_result)
+ raise
+
+
async def async_iterate(iterable: Iterable[T] | AsyncIterable[T]) -> AsyncIterator[T]:
"""
Normalize a sync or async iterable into an async iterator.
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.py
index bd764dc8e22b..3bddc7fcf6eb 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/core/test/test_async_iterate.py
@@ -5,7 +5,10 @@ import contextvars
import pytest
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import async_iterate
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import (
+ async_iterate,
+ run_in_source_executor,
+)
_probe: contextvars.ContextVar[str | None] = contextvars.ContextVar("probe", default=None)
@@ -92,3 +95,43 @@ async def test_async_iterate_close_runs_when_cancelled_mid_iteration():
# close() must have run despite cancellation mid-iteration.
assert closed_flag.wait(timeout=3.0), "iterator.close() was not called"
+
+
+async def test_run_in_source_executor_closes_the_result_of_a_cancelled_call():
+ started = threading.Event()
+ release = threading.Event()
+ closed = threading.Event()
+ # Holding a reference stops garbage collection from closing the generator for us.
+ built: list = []
+
+ def build():
+ started.set()
+ release.wait(timeout=5)
+
+ def gen():
+ try:
+ yield 1
+ finally:
+ closed.set()
+
+ it = gen()
+ next(it)
+ built.append(it)
+ return it
+
+ task = asyncio.create_task(run_in_source_executor(build))
+ await asyncio.to_thread(started.wait, 5)
+ task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+
+ release.set()
+ assert await asyncio.to_thread(closed.wait, 5)
+
+
+async def test_run_in_source_executor_propagates_contextvars():
+ _probe.set("bound-value")
+
+ assert await run_in_source_executor(_probe.get) == "bound-value"
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
index 453834bd82a1..107e55e286a4 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/pipeline.py
@@ -86,7 +86,10 @@ from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.arr
reconcile_batch_to_accumulated_schema,
source_uses_delta_write_column_selection,
)
-from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import async_iterate
+from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.async_iterate import (
+ async_iterate,
+ run_in_source_executor,
+)
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.batcher import Batcher
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.maintenance import DeltaMaintenance
from products.warehouse_sources.backend.temporal.data_imports.pipelines.core.delta.table import DeltaTableRef
@@ -998,7 +1001,10 @@ class PipelineV3(Generic[ResumableData]):
):
await self._record_table_rebuild(self._run_uuid)
- items = self._resource.items()
+ # A source can do blocking work when it builds its items, such as the DNS lookup of a
+ # host check. The source thread pool keeps that work off the event loop, as iteration
+ # already is, without tying up the default executor.
+ items = await run_in_source_executor(self._resource.items)
safe_point_scope = self._activate_safe_point(items)
source_items = self._source_items(items, source_type)
awaiting_source = True
diff --git a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
index bbc55b25c6ce..3cbda075a725 100644
--- a/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
+++ b/products/warehouse_sources/backend/temporal/data_imports/pipelines/pipeline_v3/test_pipeline.py
@@ -3098,6 +3098,35 @@ def _staged_ids(pipeline: PipelineV3) -> list[str]:
]
+async def test_the_source_builds_its_items_off_the_event_loop() -> None:
+ loop_thread = threading.get_ident()
+ built_in: list[int] = []
+
+ def items():
+ built_in.append(threading.get_ident())
+ return iter([pa.table({"id": ["a"]})])
+
+ pipeline = _runnable_pipeline(_manager(), items)
+ pipeline._finalize = AsyncMock() # type: ignore[method-assign]
+ with ExitStack() as stack:
+ for name in (
+ "reset_rows_synced_if_needed",
+ "setup_row_tracking_with_billing_check",
+ "handle_reset_or_full_refresh",
+ "handle_corrupted_delta_log",
+ ):
+ stack.enter_context(patch(f"{_PIPELINE}.{name}", new_callable=AsyncMock))
+ for name in ("validate_incremental_sync", "record_source_item_stats"):
+ stack.enter_context(patch(f"{_PIPELINE}.{name}"))
+ stack.enter_context(patch.object(ResumableSourceManager, "_get_redis", lambda self: nullcontext(MagicMock())))
+ stack.enter_context(patch(f"{_PIPELINE}.activity")).in_activity.return_value = False
+ await pipeline.run()
+
+ assert len(built_in) == 1
+ assert built_in[0] != loop_thread
+
+
class TestSourcePreemption:
@pytest.mark.parametrize("late_write", ["commit", "clear_state"])
@pytest.mark.asyncio