Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(aio): run summarization calls on the slot-sized thread pool (#119932)

Bernat Torres363f72f8e495
diff --git a/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.py b/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.py
index 0eb650cca304..2dc4418b61dc 100644
--- a/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.py
+++ b/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.py
@@ -13,7 +13,6 @@ from posthog.dataclasses import frozen
from posthog.hogql_queries.ai.ai_table_resolver import query_ai_events
from posthog.models.team import Team
from posthog.redis import get_async_client
-from posthog.sync import database_sync_to_async
from posthog.temporal.ai_observability.trace_summarization.constants import MAX_RAW_TRACE_SIZE
from posthog.temporal.ai_observability.trace_summarization.models import (
FetchAndFormatInput,
@@ -24,6 +23,7 @@ from posthog.temporal.ai_observability.trace_summarization.queries import fetch_
from posthog.temporal.ai_observability.trace_summarization.state import generate_redis_key, store_text_repr
from posthog.temporal.ai_observability.trace_summarization.utils import format_datetime_for_clickhouse
from posthog.temporal.common.heartbeat import Heartbeater
+from posthog.temporal.common.utils import on_asyncify_pool
from products.ai_observability.backend.text_repr.formatters import (
FormatterOptions,
@@ -335,7 +335,7 @@ async def fetch_and_format_activity(input: FetchAndFormatInput) -> FetchAndForma
t0 = time.monotonic()
if input.generation_id:
- result = await database_sync_to_async(_fetch_and_format_generation, thread_sensitive=False)(
+ result = await on_asyncify_pool(_fetch_and_format_generation)(
input.generation_id,
input.trace_id,
input.team_id,
@@ -344,7 +344,7 @@ async def fetch_and_format_activity(input: FetchAndFormatInput) -> FetchAndForma
input.max_length,
)
else:
- result = await database_sync_to_async(_fetch_and_format_trace, thread_sensitive=False)(
+ result = await on_asyncify_pool(_fetch_and_format_trace)(
input.trace_id, input.team_id, input.window_start, input.window_end, input.max_length
)
diff --git a/posthog/temporal/ai_observability/trace_summarization/sampling.py b/posthog/temporal/ai_observability/trace_summarization/sampling.py
index 137e6ff31f26..0c8563d27f43 100644
--- a/posthog/temporal/ai_observability/trace_summarization/sampling.py
+++ b/posthog/temporal/ai_observability/trace_summarization/sampling.py
@@ -18,7 +18,6 @@ from posthog.hogql.query import execute_hogql_query
from posthog.clickhouse.query_tagging import Feature, Product, tags_context
from posthog.models.team import Team
-from posthog.sync import database_sync_to_async
from posthog.temporal.ai_observability.trace_summarization.constants import (
AI_EVENT_TYPES,
MAX_TRACE_EVENTS_LIMIT,
@@ -27,6 +26,7 @@ from posthog.temporal.ai_observability.trace_summarization.constants import (
from posthog.temporal.ai_observability.trace_summarization.models import BatchSummarizationInputs, SampledItem
from posthog.temporal.ai_observability.trace_summarization.utils import format_datetime_for_clickhouse
from posthog.temporal.common.heartbeat import Heartbeater
+from posthog.temporal.common.utils import on_asyncify_pool
from products.cohorts.backend.models.cohort import Cohort
@@ -260,7 +260,7 @@ async def sample_items_in_window_activity(inputs: BatchSummarizationInputs) -> l
]
async with Heartbeater():
- items = await database_sync_to_async(_sample_items, thread_sensitive=False)(
+ items = await on_asyncify_pool(_sample_items)(
inputs.team_id,
inputs.window_start,
inputs.window_end,
diff --git a/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py b/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py
index 1d76f2b187fa..e1a2133daff1 100644
--- a/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py
+++ b/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py
@@ -10,7 +10,6 @@ import temporalio
from posthog.models.event.util import create_event
from posthog.models.team import Team
from posthog.redis import get_async_client
-from posthog.sync import database_sync_to_async
from posthog.temporal.ai_observability.trace_summarization import constants
from posthog.temporal.ai_observability.trace_summarization.models import (
SummarizationActivityResult,
@@ -19,6 +18,7 @@ from posthog.temporal.ai_observability.trace_summarization.models import (
)
from posthog.temporal.ai_observability.trace_summarization.state import delete_text_repr, load_text_repr
from posthog.temporal.common.heartbeat import Heartbeater
+from posthog.temporal.common.utils import on_asyncify_pool
from products.ai_observability.backend.summarization.llm import summarize
from products.ai_observability.backend.summarization.llm.schema import SummarizationResponse
@@ -209,7 +209,7 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summariza
model_enum = OpenAIModel(input.model) if input.model else None
t0 = time.monotonic()
- summary_result = await database_sync_to_async(summarize, thread_sensitive=False)(
+ summary_result = await on_asyncify_pool(summarize)(
text_repr=text_repr,
team_id=input.team_id,
mode=mode_enum,
@@ -230,7 +230,7 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summariza
)
# Step 3: Save event to ClickHouse
- team = await database_sync_to_async(Team.objects.get, thread_sensitive=False)(id=input.team_id)
+ team = await on_asyncify_pool(Team.objects.get)(id=input.team_id)
t0 = time.monotonic()
save_ctx = SaveSummaryEventContext(
summary_result=summary_result,
@@ -246,11 +246,9 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summariza
)
if is_generation:
assert input.generation_id is not None
- await database_sync_to_async(_save_generation_summary_event, thread_sensitive=False)(
- save_ctx, input.generation_id
- )
+ await on_asyncify_pool(_save_generation_summary_event)(save_ctx, input.generation_id)
else:
- await database_sync_to_async(_save_trace_summary_event, thread_sensitive=False)(save_ctx, input.event_count)
+ await on_asyncify_pool(_save_trace_summary_event)(save_ctx, input.event_count)
save_duration_s = time.monotonic() - t0
# Step 4: Request embedding
@@ -260,11 +258,11 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summariza
try:
if is_generation:
assert input.generation_id is not None
- await database_sync_to_async(_embed_generation_summary, thread_sensitive=False)(
+ await on_asyncify_pool(_embed_generation_summary)(
summary_result, input.generation_id, input.mode, input.batch_run_id, input.job_id, team
)
else:
- await database_sync_to_async(_embed_trace_summary, thread_sensitive=False)(
+ await on_asyncify_pool(_embed_trace_summary)(
summary_result, input.trace_id, input.mode, input.batch_run_id, input.job_id, team
)
embedding_requested = True
diff --git a/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py b/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py
index 131e979fc216..fd6c43aaa13d 100644
--- a/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py
+++ b/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py
@@ -1,6 +1,7 @@
"""Tests for unified summarize_and_save activity."""
import uuid
+import threading
import dataclasses
from contextlib import asynccontextmanager
from datetime import UTC, datetime
@@ -185,6 +186,37 @@ class TestSummarizeAndSaveActivity:
}
assert embed_kwargs["document_id"] == f"{generation_id}::{input_data.job_id}"
+ @pytest.mark.django_db(transaction=True)
+ @pytest.mark.asyncio
+ async def test_llm_call_runs_on_the_slot_sized_pool(self, mock_team, asyncify_executor):
+ threads: list[str] = []
+
+ def record_thread(**kwargs):
+ threads.append(threading.current_thread().name)
+ return _make_mock_summary()
+
+ asyncify_executor(2)
+ with (
+ patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.get_async_client"),
+ patch(
+ "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.load_text_repr",
+ return_value="text_repr content",
+ ),
+ patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.delete_text_repr"),
+ patch(
+ "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.summarize",
+ side_effect=record_thread,
+ ),
+ patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.create_event"),
+ patch(
+ "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.LLMTracesSummarizerEmbedder"
+ ),
+ ):
+ await _run_activity(_make_input(mock_team))
+
+ assert len(threads) == 1
+ assert threads[0].startswith("asyncify")
+
@pytest.mark.django_db(transaction=True)
@pytest.mark.asyncio
async def test_embedding_failure_captured_not_fatal(self, mock_team):
diff --git a/posthog/temporal/common/utils.py b/posthog/temporal/common/utils.py
index 88625a95c6e4..a63f548b1010 100644
--- a/posthog/temporal/common/utils.py
+++ b/posthog/temporal/common/utils.py
@@ -5,7 +5,7 @@ import threading
from collections.abc import Callable, Coroutine
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime
-from functools import wraps
+from functools import partial, wraps
from typing import Any, ParamSpec, TypeVar, cast
import django.db
@@ -14,6 +14,7 @@ from django.conf import settings
from asgiref.sync import sync_to_async
from temporalio import activity, workflow
+from posthog.sync import database_sync_to_async_pool
from posthog.temporal.common.heartbeat_sync import heartbeat_through_loop
P = ParamSpec("P")
@@ -102,6 +103,46 @@ def make_sync_retryable_with_exponential_backoff(
return inner
+def _log_slow_thread_call(call: Callable[[], T], name: str, submit_time: float) -> Callable[[], T]:
+ """Wrap a thread-pool call so a slow thread wait or execution logs an `asyncify_slow` warning."""
+
+ def instrumented() -> T:
+ start_time = time.monotonic()
+ try:
+ return call()
+ finally:
+ now = time.monotonic()
+ thread_wait = start_time - submit_time
+ execution_time = now - start_time
+ if activity.in_activity() and max(thread_wait, execution_time) >= ASYNCIFY_SLOW_THRESHOLD_SECONDS:
+ # The structlog chain has no ExtraAdder, so stdlib `extra=` fields never reach the log.
+ activity.logger.warning(
+ f"asyncify_slow function={name} "
+ f"thread_wait_seconds={thread_wait:.3f} execution_seconds={execution_time:.3f} "
+ f"thread_name={threading.current_thread().name}"
+ )
+
+ return instrumented
+
+
+def on_asyncify_pool(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:
+ """Run a blocking call from an async activity on the pool `@asyncify` uses, sized to the worker's slots.
+
+ Use it for ``await`` steps inside an async activity. ``database_sync_to_async(..., thread_sensitive=False)``
+ without an executor runs on the event loop's default pool instead, which CPython caps at the node's
+ CPU count + 4, so slots above that cap wait for a thread. The pool is read on each call, so a
+ module-level ``@on_asyncify_pool`` still picks up the pool the worker configures at startup.
+ """
+
+ @wraps(fn)
+ async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
+ name = getattr(fn, "__name__", type(fn).__name__)
+ instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), name, time.monotonic())
+ return await database_sync_to_async_pool(instrumented, executor=get_asyncify_executor())()
+
+ return wrapper
+
+
def asyncify(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:
"""Decorator to convert a sync function using sync_to_async - this preserves type hints for Temporal's serialization while allowing sync Django ORM code.
@@ -122,23 +163,7 @@ def asyncify(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:
@wraps(fn)
async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
- submit_time = time.monotonic()
-
- def instrumented() -> T:
- start_time = time.monotonic()
- try:
- return fn(*args, **kwargs)
- finally:
- now = time.monotonic()
- thread_wait = start_time - submit_time
- execution_time = now - start_time
- if activity.in_activity() and max(thread_wait, execution_time) >= ASYNCIFY_SLOW_THRESHOLD_SECONDS:
- # The structlog chain has no ExtraAdder, so stdlib `extra=` fields never reach the log.
- activity.logger.warning(
- f"asyncify_slow function={fn.__name__} "
- f"thread_wait_seconds={thread_wait:.3f} execution_seconds={execution_time:.3f} "
- f"thread_name={threading.current_thread().name}"
- )
+ instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), fn.__name__, time.monotonic())
run_on_thread = sync_to_async(thread_sensitive=False, executor=get_asyncify_executor())(
close_db_connections(instrumented)
diff --git a/posthog/temporal/conftest.py b/posthog/temporal/conftest.py
index 6b1cc2afb271..8040c7fab7db 100644
--- a/posthog/temporal/conftest.py
+++ b/posthog/temporal/conftest.py
@@ -2,6 +2,7 @@ from collections.abc import Iterator
import pytest
+from posthog.temporal.tests.utils.asyncify import ConfigureExecutor, restored_asyncify_executor
from posthog.temporal.tests.utils.logger import restored_structlog_config
@@ -9,3 +10,9 @@ from posthog.temporal.tests.utils.logger import restored_structlog_config
def _restore_structlog_config() -> Iterator[None]:
with restored_structlog_config():
yield
+
+
+def asyncify_executor() -> Iterator[ConfigureExecutor]:
+ with restored_asyncify_executor() as configure:
+ yield configure
diff --git a/posthog/temporal/tests/test_asyncify.py b/posthog/temporal/tests/test_asyncify.py
index 19a20d27c4f6..2a57e4d5236f 100644
--- a/posthog/temporal/tests/test_asyncify.py
+++ b/posthog/temporal/tests/test_asyncify.py
@@ -1,10 +1,8 @@
import asyncio
import threading
import dataclasses
-from collections.abc import Iterator
-from concurrent.futures import ThreadPoolExecutor
from datetime import timedelta
-from typing import Any, Protocol
+from typing import Any
import pytest
from unittest import mock
@@ -15,27 +13,23 @@ from temporalio.testing import ActivityEnvironment
from posthog.temporal.common import utils, worker
from posthog.temporal.common.heartbeat_sync import HeartbeaterSync
-from posthog.temporal.common.utils import asyncify, configure_asyncify_executor, get_asyncify_executor
+from posthog.temporal.common.utils import asyncify, get_asyncify_executor, on_asyncify_pool
+from posthog.temporal.tests.utils.asyncify import ConfigureExecutor
pytestmark = pytest.mark.asyncio
BARRIER_TIMEOUT_SECONDS = 5
-class ConfigureExecutor(Protocol):
- def __call__(self, max_workers: int) -> ThreadPoolExecutor: ...
+@on_asyncify_pool
+def _thread_name_decorated_at_import() -> str:
+ return threading.current_thread().name
-def asyncify_executor() -> Iterator[ConfigureExecutor]:
- previous = get_asyncify_executor()
- try:
- yield configure_asyncify_executor
- finally:
- configured = get_asyncify_executor()
- if configured is not previous and configured is not None:
- configured.shutdown(wait=False)
- utils._asyncify_executor = previous
+async def test_on_asyncify_pool_uses_the_pool_configured_after_decoration(asyncify_executor: ConfigureExecutor) -> None:
+ asyncify_executor(2)
+
+ assert (await _thread_name_decorated_at_import()).startswith("asyncify")
async def test_asyncify_runs_on_the_configured_executor(asyncify_executor: ConfigureExecutor) -> None:
diff --git a/posthog/temporal/tests/utils/asyncify.py b/posthog/temporal/tests/utils/asyncify.py
new file mode 100644
index 000000000000..f2c8dab2fb53
--- /dev/null
+++ b/posthog/temporal/tests/utils/asyncify.py
@@ -0,0 +1,24 @@
+from collections.abc import Iterator
+from concurrent.futures import ThreadPoolExecutor
+from contextlib import contextmanager
+from typing import Protocol
+
+from posthog.temporal.common import utils
+from posthog.temporal.common.utils import configure_asyncify_executor, get_asyncify_executor
+
+
+class ConfigureExecutor(Protocol):
+ def __call__(self, max_workers: int) -> ThreadPoolExecutor: ...
+
+
+@contextmanager
+def restored_asyncify_executor() -> Iterator[ConfigureExecutor]:
+ """Let a test configure the worker's asyncify pool, then put back whatever pool was set before."""
+ previous = get_asyncify_executor()
+ try:
+ yield configure_asyncify_executor
+ finally:
+ configured = get_asyncify_executor()
+ if configured is not previous and configured is not None:
+ configured.shutdown(wait=False)
+ utils._asyncify_executor = previous