Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): warn only on slow thread waits in on_asyncify_pool (#120637)

Bernat Torrese646a824909a
diff --git a/posthog/temporal/common/utils.py b/posthog/temporal/common/utils.py
index a63f548b1010..b780bc7cece8 100644
--- a/posthog/temporal/common/utils.py
+++ b/posthog/temporal/common/utils.py
@@ -103,8 +103,10 @@ def make_sync_retryable_with_exponential_backoff(
return inner
-def _log_slow_thread_call(call: Callable[[], T], name: str, submit_time: float) -> Callable[[], T]:
- """Wrap a thread-pool call so a slow thread wait or execution logs an `asyncify_slow` warning."""
+def _log_slow_thread_call(
+ call: Callable[[], T], name: str, submit_time: float, *, log_slow_execution: bool
+) -> Callable[[], T]:
+ """Wrap a thread-pool call so a slow thread wait, or a slow execution when asked, logs an `asyncify_slow` warning."""
def instrumented() -> T:
start_time = time.monotonic()
@@ -114,7 +116,10 @@ def _log_slow_thread_call(call: Callable[[], T], name: str, submit_time: float)
now = time.monotonic()
thread_wait = start_time - submit_time
execution_time = now - start_time
- if activity.in_activity() and max(thread_wait, execution_time) >= ASYNCIFY_SLOW_THRESHOLD_SECONDS:
+ slow = thread_wait >= ASYNCIFY_SLOW_THRESHOLD_SECONDS or (
+ log_slow_execution and execution_time >= ASYNCIFY_SLOW_THRESHOLD_SECONDS
+ )
+ if activity.in_activity() and slow:
# The structlog chain has no ExtraAdder, so stdlib `extra=` fields never reach the log.
activity.logger.warning(
f"asyncify_slow function={name} "
@@ -132,12 +137,17 @@ def on_asyncify_pool(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:
without an executor runs on the event loop's default pool instead, which CPython caps at the node's
CPU count + 4, so slots above that cap wait for a thread. The pool is read on each call, so a
module-level ``@on_asyncify_pool`` still picks up the pool the worker configures at startup.
+
+ It logs ``asyncify_slow`` only for a slow thread wait. These steps are often long LLM or ClickHouse
+ calls, so a slow execution is normal and would log on almost every call.
"""
@wraps(fn)
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
name = getattr(fn, "__name__", type(fn).__name__)
- instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), name, time.monotonic())
+ instrumented = _log_slow_thread_call(
+ partial(fn, *args, **kwargs), name, time.monotonic(), log_slow_execution=False
+ )
return await database_sync_to_async_pool(instrumented, executor=get_asyncify_executor())()
return wrapper
@@ -163,7 +173,9 @@ def asyncify(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:
@wraps(fn)
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
- instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), fn.__name__, time.monotonic())
+ instrumented = _log_slow_thread_call(
+ partial(fn, *args, **kwargs), fn.__name__, time.monotonic(), log_slow_execution=True
+ )
run_on_thread = sync_to_async(thread_sensitive=False, executor=get_asyncify_executor())(
close_db_connections(instrumented)
diff --git a/posthog/temporal/tests/test_asyncify.py b/posthog/temporal/tests/test_asyncify.py
index 2a57e4d5236f..b13e97b9c127 100644
--- a/posthog/temporal/tests/test_asyncify.py
+++ b/posthog/temporal/tests/test_asyncify.py
@@ -32,6 +32,30 @@ async def test_on_asyncify_pool_uses_the_pool_configured_after_decoration(asynci
assert (await _thread_name_decorated_at_import()).startswith("asyncify")
+ "helper,submit_start_end,expect_warning",
+ [
+ ("on_asyncify_pool", (0.0, 0.0, 5.0), False),
+ ("on_asyncify_pool", (0.0, 5.0, 5.0), True),
+ ("asyncify", (0.0, 0.0, 5.0), True),
+ ("asyncify", (0.0, 5.0, 5.0), True),
+ ],
+)
+async def test_slow_call_warning(
+ caplog: pytest.LogCaptureFixture, helper: str, submit_start_end: tuple[float, float, float], expect_warning: bool
+) -> None:
+ def blocking_call() -> None:
+ return None
+
+ wrapped = on_asyncify_pool(blocking_call) if helper == "on_asyncify_pool" else asyncify(blocking_call)
+ clock = mock.Mock(monotonic=mock.Mock(side_effect=submit_start_end))
+
+ with mock.patch.object(utils, "time", clock), caplog.at_level("WARNING"):
+ await ActivityEnvironment().run(wrapped)
+
+ assert any("asyncify_slow" in record.getMessage() for record in caplog.records) is expect_warning
+
+
async def test_asyncify_runs_on_the_configured_executor(asyncify_executor: ConfigureExecutor) -> None:
asyncify_executor(2)