:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): warn only on slow thread waits in on_asyncify_pool (#120637)
Bernat Torrese646a824909a
diff --git a/posthog/temporal/common/utils.py b/posthog/temporal/common/utils.pyindex a63f548b1010..b780bc7cece8 100644--- a/posthog/temporal/common/utils.py+++ b/posthog/temporal/common/utils.py@@ -103,8 +103,10 @@ def make_sync_retryable_with_exponential_backoff(return inner-def _log_slow_thread_call(call: Callable[[], T], name: str, submit_time: float) -> Callable[[], T]:- """Wrap a thread-pool call so a slow thread wait or execution logs an `asyncify_slow` warning."""+def _log_slow_thread_call(+ call: Callable[[], T], name: str, submit_time: float, *, log_slow_execution: bool+) -> Callable[[], T]:+ """Wrap a thread-pool call so a slow thread wait, or a slow execution when asked, logs an `asyncify_slow` warning."""def instrumented() -> T:start_time = time.monotonic()@@ -114,7 +116,10 @@ def _log_slow_thread_call(call: Callable[[], T], name: str, submit_time: float)now = time.monotonic()thread_wait = start_time - submit_timeexecution_time = now - start_time- if activity.in_activity() and max(thread_wait, execution_time) >= ASYNCIFY_SLOW_THRESHOLD_SECONDS:+ slow = thread_wait >= ASYNCIFY_SLOW_THRESHOLD_SECONDS or (+ log_slow_execution and execution_time >= ASYNCIFY_SLOW_THRESHOLD_SECONDS+ )+ if activity.in_activity() and slow:# The structlog chain has no ExtraAdder, so stdlib `extra=` fields never reach the log.activity.logger.warning(f"asyncify_slow function={name} "@@ -132,12 +137,17 @@ def on_asyncify_pool(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:without an executor runs on the event loop's default pool instead, which CPython caps at the node'sCPU count + 4, so slots above that cap wait for a thread. The pool is read on each call, so amodule-level ``@on_asyncify_pool`` still picks up the pool the worker configures at startup.++ It logs ``asyncify_slow`` only for a slow thread wait. These steps are often long LLM or ClickHouse+ calls, so a slow execution is normal and would log on almost every call."""@wraps(fn)async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:name = getattr(fn, "__name__", type(fn).__name__)- instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), name, time.monotonic())+ instrumented = _log_slow_thread_call(+ partial(fn, *args, **kwargs), name, time.monotonic(), log_slow_execution=False+ )return await database_sync_to_async_pool(instrumented, executor=get_asyncify_executor())()return wrapper@@ -163,7 +173,9 @@ def asyncify(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:@wraps(fn)async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:- instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), fn.__name__, time.monotonic())+ instrumented = _log_slow_thread_call(+ partial(fn, *args, **kwargs), fn.__name__, time.monotonic(), log_slow_execution=True+ )run_on_thread = sync_to_async(thread_sensitive=False, executor=get_asyncify_executor())(close_db_connections(instrumented)diff --git a/posthog/temporal/tests/test_asyncify.py b/posthog/temporal/tests/test_asyncify.pyindex 2a57e4d5236f..b13e97b9c127 100644--- a/posthog/temporal/tests/test_asyncify.py+++ b/posthog/temporal/tests/test_asyncify.py@@ -32,6 +32,30 @@ async def test_on_asyncify_pool_uses_the_pool_configured_after_decoration(asynciassert (await _thread_name_decorated_at_import()).startswith("asyncify")+ "helper,submit_start_end,expect_warning",+ [+ ("on_asyncify_pool", (0.0, 0.0, 5.0), False),+ ("on_asyncify_pool", (0.0, 5.0, 5.0), True),+ ("asyncify", (0.0, 0.0, 5.0), True),+ ("asyncify", (0.0, 5.0, 5.0), True),+ ],+)+async def test_slow_call_warning(+ caplog: pytest.LogCaptureFixture, helper: str, submit_start_end: tuple[float, float, float], expect_warning: bool+) -> None:+ def blocking_call() -> None:+ return None++ wrapped = on_asyncify_pool(blocking_call) if helper == "on_asyncify_pool" else asyncify(blocking_call)+ clock = mock.Mock(monotonic=mock.Mock(side_effect=submit_start_end))++ with mock.patch.object(utils, "time", clock), caplog.at_level("WARNING"):+ await ActivityEnvironment().run(wrapped)++ assert any("asyncify_slow" in record.getMessage() for record in caplog.records) is expect_warning++async def test_asyncify_runs_on_the_configured_executor(asyncify_executor: ConfigureExecutor) -> None:asyncify_executor(2)