:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(aio): run summarization calls on the slot-sized thread pool (#119932)
Bernat Torres363f72f8e495
diff --git a/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.py b/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.pyindex 0eb650cca304..2dc4418b61dc 100644--- a/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.py+++ b/posthog/temporal/ai_observability/trace_summarization/fetch_and_format.py@@ -13,7 +13,6 @@ from posthog.dataclasses import frozenfrom posthog.hogql_queries.ai.ai_table_resolver import query_ai_eventsfrom posthog.models.team import Teamfrom posthog.redis import get_async_client-from posthog.sync import database_sync_to_asyncfrom posthog.temporal.ai_observability.trace_summarization.constants import MAX_RAW_TRACE_SIZEfrom posthog.temporal.ai_observability.trace_summarization.models import (FetchAndFormatInput,@@ -24,6 +23,7 @@ from posthog.temporal.ai_observability.trace_summarization.queries import fetch_from posthog.temporal.ai_observability.trace_summarization.state import generate_redis_key, store_text_reprfrom posthog.temporal.ai_observability.trace_summarization.utils import format_datetime_for_clickhousefrom posthog.temporal.common.heartbeat import Heartbeater+from posthog.temporal.common.utils import on_asyncify_poolfrom products.ai_observability.backend.text_repr.formatters import (FormatterOptions,@@ -335,7 +335,7 @@ async def fetch_and_format_activity(input: FetchAndFormatInput) -> FetchAndFormat0 = time.monotonic()if input.generation_id:- result = await database_sync_to_async(_fetch_and_format_generation, thread_sensitive=False)(+ result = await on_asyncify_pool(_fetch_and_format_generation)(input.generation_id,input.trace_id,input.team_id,@@ -344,7 +344,7 @@ async def fetch_and_format_activity(input: FetchAndFormatInput) -> FetchAndFormainput.max_length,)else:- result = await database_sync_to_async(_fetch_and_format_trace, thread_sensitive=False)(+ result = await on_asyncify_pool(_fetch_and_format_trace)(input.trace_id, input.team_id, input.window_start, input.window_end, input.max_length)diff --git a/posthog/temporal/ai_observability/trace_summarization/sampling.py b/posthog/temporal/ai_observability/trace_summarization/sampling.pyindex 137e6ff31f26..0c8563d27f43 100644--- a/posthog/temporal/ai_observability/trace_summarization/sampling.py+++ b/posthog/temporal/ai_observability/trace_summarization/sampling.py@@ -18,7 +18,6 @@ from posthog.hogql.query import execute_hogql_queryfrom posthog.clickhouse.query_tagging import Feature, Product, tags_contextfrom posthog.models.team import Team-from posthog.sync import database_sync_to_asyncfrom posthog.temporal.ai_observability.trace_summarization.constants import (AI_EVENT_TYPES,MAX_TRACE_EVENTS_LIMIT,@@ -27,6 +26,7 @@ from posthog.temporal.ai_observability.trace_summarization.constants import (from posthog.temporal.ai_observability.trace_summarization.models import BatchSummarizationInputs, SampledItemfrom posthog.temporal.ai_observability.trace_summarization.utils import format_datetime_for_clickhousefrom posthog.temporal.common.heartbeat import Heartbeater+from posthog.temporal.common.utils import on_asyncify_poolfrom products.cohorts.backend.models.cohort import Cohort@@ -260,7 +260,7 @@ async def sample_items_in_window_activity(inputs: BatchSummarizationInputs) -> l]async with Heartbeater():- items = await database_sync_to_async(_sample_items, thread_sensitive=False)(+ items = await on_asyncify_pool(_sample_items)(inputs.team_id,inputs.window_start,inputs.window_end,diff --git a/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py b/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.pyindex 1d76f2b187fa..e1a2133daff1 100644--- a/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py+++ b/posthog/temporal/ai_observability/trace_summarization/summarize_and_save.py@@ -10,7 +10,6 @@ import temporaliofrom posthog.models.event.util import create_eventfrom posthog.models.team import Teamfrom posthog.redis import get_async_client-from posthog.sync import database_sync_to_asyncfrom posthog.temporal.ai_observability.trace_summarization import constantsfrom posthog.temporal.ai_observability.trace_summarization.models import (SummarizationActivityResult,@@ -19,6 +18,7 @@ from posthog.temporal.ai_observability.trace_summarization.models import ()from posthog.temporal.ai_observability.trace_summarization.state import delete_text_repr, load_text_reprfrom posthog.temporal.common.heartbeat import Heartbeater+from posthog.temporal.common.utils import on_asyncify_poolfrom products.ai_observability.backend.summarization.llm import summarizefrom products.ai_observability.backend.summarization.llm.schema import SummarizationResponse@@ -209,7 +209,7 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summarizamodel_enum = OpenAIModel(input.model) if input.model else Nonet0 = time.monotonic()- summary_result = await database_sync_to_async(summarize, thread_sensitive=False)(+ summary_result = await on_asyncify_pool(summarize)(text_repr=text_repr,team_id=input.team_id,mode=mode_enum,@@ -230,7 +230,7 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summariza)# Step 3: Save event to ClickHouse- team = await database_sync_to_async(Team.objects.get, thread_sensitive=False)(id=input.team_id)+ team = await on_asyncify_pool(Team.objects.get)(id=input.team_id)t0 = time.monotonic()save_ctx = SaveSummaryEventContext(summary_result=summary_result,@@ -246,11 +246,9 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summariza)if is_generation:assert input.generation_id is not None- await database_sync_to_async(_save_generation_summary_event, thread_sensitive=False)(- save_ctx, input.generation_id- )+ await on_asyncify_pool(_save_generation_summary_event)(save_ctx, input.generation_id)else:- await database_sync_to_async(_save_trace_summary_event, thread_sensitive=False)(save_ctx, input.event_count)+ await on_asyncify_pool(_save_trace_summary_event)(save_ctx, input.event_count)save_duration_s = time.monotonic() - t0# Step 4: Request embedding@@ -260,11 +258,11 @@ async def summarize_and_save_activity(input: SummarizeAndSaveInput) -> Summarizatry:if is_generation:assert input.generation_id is not None- await database_sync_to_async(_embed_generation_summary, thread_sensitive=False)(+ await on_asyncify_pool(_embed_generation_summary)(summary_result, input.generation_id, input.mode, input.batch_run_id, input.job_id, team)else:- await database_sync_to_async(_embed_trace_summary, thread_sensitive=False)(+ await on_asyncify_pool(_embed_trace_summary)(summary_result, input.trace_id, input.mode, input.batch_run_id, input.job_id, team)embedding_requested = Truediff --git a/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py b/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.pyindex 131e979fc216..fd6c43aaa13d 100644--- a/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py+++ b/posthog/temporal/ai_observability/trace_summarization/tests/test_summarize_and_save.py@@ -1,6 +1,7 @@"""Tests for unified summarize_and_save activity."""import uuid+import threadingimport dataclassesfrom contextlib import asynccontextmanagerfrom datetime import UTC, datetime@@ -185,6 +186,37 @@ class TestSummarizeAndSaveActivity:}assert embed_kwargs["document_id"] == f"{generation_id}::{input_data.job_id}"+ @pytest.mark.django_db(transaction=True)+ @pytest.mark.asyncio+ async def test_llm_call_runs_on_the_slot_sized_pool(self, mock_team, asyncify_executor):+ threads: list[str] = []++ def record_thread(**kwargs):+ threads.append(threading.current_thread().name)+ return _make_mock_summary()++ asyncify_executor(2)+ with (+ patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.get_async_client"),+ patch(+ "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.load_text_repr",+ return_value="text_repr content",+ ),+ patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.delete_text_repr"),+ patch(+ "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.summarize",+ side_effect=record_thread,+ ),+ patch("posthog.temporal.ai_observability.trace_summarization.summarize_and_save.create_event"),+ patch(+ "posthog.temporal.ai_observability.trace_summarization.summarize_and_save.LLMTracesSummarizerEmbedder"+ ),+ ):+ await _run_activity(_make_input(mock_team))++ assert len(threads) == 1+ assert threads[0].startswith("asyncify")+@pytest.mark.django_db(transaction=True)@pytest.mark.asyncioasync def test_embedding_failure_captured_not_fatal(self, mock_team):diff --git a/posthog/temporal/common/utils.py b/posthog/temporal/common/utils.pyindex 88625a95c6e4..a63f548b1010 100644--- a/posthog/temporal/common/utils.py+++ b/posthog/temporal/common/utils.py@@ -5,7 +5,7 @@ import threadingfrom collections.abc import Callable, Coroutinefrom concurrent.futures import ThreadPoolExecutorfrom datetime import datetime-from functools import wraps+from functools import partial, wrapsfrom typing import Any, ParamSpec, TypeVar, castimport django.db@@ -14,6 +14,7 @@ from django.conf import settingsfrom asgiref.sync import sync_to_asyncfrom temporalio import activity, workflow+from posthog.sync import database_sync_to_async_poolfrom posthog.temporal.common.heartbeat_sync import heartbeat_through_loopP = ParamSpec("P")@@ -102,6 +103,46 @@ def make_sync_retryable_with_exponential_backoff(return inner+def _log_slow_thread_call(call: Callable[[], T], name: str, submit_time: float) -> Callable[[], T]:+ """Wrap a thread-pool call so a slow thread wait or execution logs an `asyncify_slow` warning."""++ def instrumented() -> T:+ start_time = time.monotonic()+ try:+ return call()+ finally:+ now = time.monotonic()+ thread_wait = start_time - submit_time+ execution_time = now - start_time+ if activity.in_activity() and max(thread_wait, execution_time) >= ASYNCIFY_SLOW_THRESHOLD_SECONDS:+ # The structlog chain has no ExtraAdder, so stdlib `extra=` fields never reach the log.+ activity.logger.warning(+ f"asyncify_slow function={name} "+ f"thread_wait_seconds={thread_wait:.3f} execution_seconds={execution_time:.3f} "+ f"thread_name={threading.current_thread().name}"+ )++ return instrumented+++def on_asyncify_pool(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:+ """Run a blocking call from an async activity on the pool `@asyncify` uses, sized to the worker's slots.++ Use it for ``await`` steps inside an async activity. ``database_sync_to_async(..., thread_sensitive=False)``+ without an executor runs on the event loop's default pool instead, which CPython caps at the node's+ CPU count + 4, so slots above that cap wait for a thread. The pool is read on each call, so a+ module-level ``@on_asyncify_pool`` still picks up the pool the worker configures at startup.+ """++ @wraps(fn)+ async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:+ name = getattr(fn, "__name__", type(fn).__name__)+ instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), name, time.monotonic())+ return await database_sync_to_async_pool(instrumented, executor=get_asyncify_executor())()++ return wrapper++def asyncify(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:"""Decorator to convert a sync function using sync_to_async - this preserves type hints for Temporal's serialization while allowing sync Django ORM code.@@ -122,23 +163,7 @@ def asyncify(fn: Callable[P, T]) -> Callable[P, Coroutine[Any, Any, T]]:@wraps(fn)async def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:- submit_time = time.monotonic()-- def instrumented() -> T:- start_time = time.monotonic()- try:- return fn(*args, **kwargs)- finally:- now = time.monotonic()- thread_wait = start_time - submit_time- execution_time = now - start_time- if activity.in_activity() and max(thread_wait, execution_time) >= ASYNCIFY_SLOW_THRESHOLD_SECONDS:- # The structlog chain has no ExtraAdder, so stdlib `extra=` fields never reach the log.- activity.logger.warning(- f"asyncify_slow function={fn.__name__} "- f"thread_wait_seconds={thread_wait:.3f} execution_seconds={execution_time:.3f} "- f"thread_name={threading.current_thread().name}"- )+ instrumented = _log_slow_thread_call(partial(fn, *args, **kwargs), fn.__name__, time.monotonic())run_on_thread = sync_to_async(thread_sensitive=False, executor=get_asyncify_executor())(close_db_connections(instrumented)diff --git a/posthog/temporal/conftest.py b/posthog/temporal/conftest.pyindex 6b1cc2afb271..8040c7fab7db 100644--- a/posthog/temporal/conftest.py+++ b/posthog/temporal/conftest.py@@ -2,6 +2,7 @@ from collections.abc import Iteratorimport pytest+from posthog.temporal.tests.utils.asyncify import ConfigureExecutor, restored_asyncify_executorfrom posthog.temporal.tests.utils.logger import restored_structlog_config@@ -9,3 +10,9 @@ from posthog.temporal.tests.utils.logger import restored_structlog_configdef _restore_structlog_config() -> Iterator[None]:with restored_structlog_config():yield+++def asyncify_executor() -> Iterator[ConfigureExecutor]:+ with restored_asyncify_executor() as configure:+ yield configurediff --git a/posthog/temporal/tests/test_asyncify.py b/posthog/temporal/tests/test_asyncify.pyindex 19a20d27c4f6..2a57e4d5236f 100644--- a/posthog/temporal/tests/test_asyncify.py+++ b/posthog/temporal/tests/test_asyncify.py@@ -1,10 +1,8 @@import asyncioimport threadingimport dataclasses-from collections.abc import Iterator-from concurrent.futures import ThreadPoolExecutorfrom datetime import timedelta-from typing import Any, Protocol+from typing import Anyimport pytestfrom unittest import mock@@ -15,27 +13,23 @@ from temporalio.testing import ActivityEnvironmentfrom posthog.temporal.common import utils, workerfrom posthog.temporal.common.heartbeat_sync import HeartbeaterSync-from posthog.temporal.common.utils import asyncify, configure_asyncify_executor, get_asyncify_executor+from posthog.temporal.common.utils import asyncify, get_asyncify_executor, on_asyncify_pool+from posthog.temporal.tests.utils.asyncify import ConfigureExecutorpytestmark = pytest.mark.asyncioBARRIER_TIMEOUT_SECONDS = 5-class ConfigureExecutor(Protocol):- def __call__(self, max_workers: int) -> ThreadPoolExecutor: ...+@on_asyncify_pool+def _thread_name_decorated_at_import() -> str:+ return threading.current_thread().name-def asyncify_executor() -> Iterator[ConfigureExecutor]:- previous = get_asyncify_executor()- try:- yield configure_asyncify_executor- finally:- configured = get_asyncify_executor()- if configured is not previous and configured is not None:- configured.shutdown(wait=False)- utils._asyncify_executor = previous+async def test_on_asyncify_pool_uses_the_pool_configured_after_decoration(asyncify_executor: ConfigureExecutor) -> None:+ asyncify_executor(2)++ assert (await _thread_name_decorated_at_import()).startswith("asyncify")async def test_asyncify_runs_on_the_configured_executor(asyncify_executor: ConfigureExecutor) -> None:diff --git a/posthog/temporal/tests/utils/asyncify.py b/posthog/temporal/tests/utils/asyncify.pynew file mode 100644index 000000000000..f2c8dab2fb53--- /dev/null+++ b/posthog/temporal/tests/utils/asyncify.py@@ -0,0 +1,24 @@+from collections.abc import Iterator+from concurrent.futures import ThreadPoolExecutor+from contextlib import contextmanager+from typing import Protocol++from posthog.temporal.common import utils+from posthog.temporal.common.utils import configure_asyncify_executor, get_asyncify_executor+++class ConfigureExecutor(Protocol):+ def __call__(self, max_workers: int) -> ThreadPoolExecutor: ...+++@contextmanager+def restored_asyncify_executor() -> Iterator[ConfigureExecutor]:+ """Let a test configure the worker's asyncify pool, then put back whatever pool was set before."""+ previous = get_asyncify_executor()+ try:+ yield configure_asyncify_executor+ finally:+ configured = get_asyncify_executor()+ if configured is not previous and configured is not None:+ configured.shutdown(wait=False)+ utils._asyncify_executor = previous